Examples with SubscriptionsStore org.apache.distributedlog.api.subscription.SubscriptionsStore used on opensource projects

Example 1 with SubscriptionsStore

use of org.apache.distributedlog.api.subscription.SubscriptionsStore in project bookkeeper by apache.

the class TestBKDistributedLogManager method testSubscriptionsStore.

@Test(timeout = 60000)
public void testSubscriptionsStore() throws Exception {
    String name = "distrlog-subscriptions-store";
    String subscriber0 = "subscriber-0";
    String subscriber1 = "subscriber-1";
    String subscriber2 = "subscriber-2";
    DLSN commitPosition0 = new DLSN(4, 33, 5);
    DLSN commitPosition1 = new DLSN(4, 34, 5);
    DLSN commitPosition2 = new DLSN(5, 34, 5);
    DLSN commitPosition3 = new DLSN(6, 35, 6);
    DistributedLogManager dlm = createNewDLM(conf, name);
    SubscriptionsStore store = dlm.getSubscriptionsStore();
    // no data
    assertEquals(Utils.ioResult(store.getLastCommitPosition(subscriber0)), DLSN.NonInclusiveLowerBound);
    assertEquals(Utils.ioResult(store.getLastCommitPosition(subscriber1)), DLSN.NonInclusiveLowerBound);
    assertEquals(Utils.ioResult(store.getLastCommitPosition(subscriber2)), DLSN.NonInclusiveLowerBound);
    // empty
    assertTrue(Utils.ioResult(store.getLastCommitPositions()).isEmpty());
    // subscriber 0 advance
    Utils.ioResult(store.advanceCommitPosition(subscriber0, commitPosition0));
    assertEquals(commitPosition0, Utils.ioResult(store.getLastCommitPosition(subscriber0)));
    Map<String, DLSN> committedPositions = Utils.ioResult(store.getLastCommitPositions());
    assertEquals(1, committedPositions.size());
    assertEquals(commitPosition0, committedPositions.get(subscriber0));
    // subscriber 1 advance
    Utils.ioResult(store.advanceCommitPosition(subscriber1, commitPosition1));
    assertEquals(commitPosition1, Utils.ioResult(store.getLastCommitPosition(subscriber1)));
    committedPositions = Utils.ioResult(store.getLastCommitPositions());
    assertEquals(2, committedPositions.size());
    assertEquals(commitPosition0, committedPositions.get(subscriber0));
    assertEquals(commitPosition1, committedPositions.get(subscriber1));
    // subscriber 2 advance
    Utils.ioResult(store.advanceCommitPosition(subscriber2, commitPosition2));
    assertEquals(commitPosition2, Utils.ioResult(store.getLastCommitPosition(subscriber2)));
    committedPositions = Utils.ioResult(store.getLastCommitPositions());
    assertEquals(3, committedPositions.size());
    assertEquals(commitPosition0, committedPositions.get(subscriber0));
    assertEquals(commitPosition1, committedPositions.get(subscriber1));
    assertEquals(commitPosition2, committedPositions.get(subscriber2));
    // subscriber 2 advance again
    DistributedLogManager newDLM = createNewDLM(conf, name);
    SubscriptionsStore newStore = newDLM.getSubscriptionsStore();
    Utils.ioResult(newStore.advanceCommitPosition(subscriber2, commitPosition3));
    newStore.close();
    newDLM.close();
    committedPositions = Utils.ioResult(store.getLastCommitPositions());
    assertEquals(3, committedPositions.size());
    assertEquals(commitPosition0, committedPositions.get(subscriber0));
    assertEquals(commitPosition1, committedPositions.get(subscriber1));
    assertEquals(commitPosition3, committedPositions.get(subscriber2));
    dlm.close();
}

Example 2 with SubscriptionsStore

use of org.apache.distributedlog.api.subscription.SubscriptionsStore in project bookkeeper by apache.

the class TestAsyncReaderLock method testAsyncReadWithSubscriberId.

@Test(timeout = 60000)
public void testAsyncReadWithSubscriberId() throws Exception {
    String name = "distrlog-asyncread-with-sbuscriber-id";
    String subscriberId = "asyncreader";
    DistributedLogConfiguration confLocal = new DistributedLogConfiguration();
    confLocal.addConfiguration(conf);
    confLocal.setOutputBufferSize(0);
    confLocal.setImmediateFlushEnabled(true);
    DistributedLogManager dlm = createNewDLM(confLocal, name);
    DLSN readDLSN = DLSN.InitialDLSN;
    int txid = 1;
    for (long i = 0; i < 3; i++) {
        BKAsyncLogWriter writer = (BKAsyncLogWriter) dlm.startAsyncLogSegmentNonPartitioned();
        for (long j = 1; j <= 10; j++) {
            DLSN dlsn = Utils.ioResult(writer.write(DLMTestUtil.getEmptyLogRecordInstance(txid++)));
            if (i == 1 && j == 1L) {
                readDLSN = dlsn;
            }
        }
        writer.closeAndComplete();
    }
    BKAsyncLogReader reader0 = (BKAsyncLogReader) Utils.ioResult(dlm.getAsyncLogReaderWithLock(subscriberId));
    assertEquals(DLSN.NonInclusiveLowerBound, reader0.getStartDLSN());
    long numTxns = 0;
    LogRecordWithDLSN record = Utils.ioResult(reader0.readNext());
    while (null != record) {
        DLMTestUtil.verifyEmptyLogRecord(record);
        ++numTxns;
        assertEquals(numTxns, record.getTransactionId());
        assertEquals(record.getTransactionId() - 1, record.getSequenceId());
        if (txid - 1 == numTxns) {
            break;
        }
        record = Utils.ioResult(reader0.readNext());
    }
    assertEquals(txid - 1, numTxns);
    Utils.close(reader0);
    SubscriptionsStore subscriptionsStore = dlm.getSubscriptionsStore();
    Utils.ioResult(subscriptionsStore.advanceCommitPosition(subscriberId, readDLSN));
    BKAsyncLogReader reader1 = (BKAsyncLogReader) Utils.ioResult(dlm.getAsyncLogReaderWithLock(subscriberId));
    assertEquals(readDLSN, reader1.getStartDLSN());
    numTxns = 0;
    long startTxID = 10L;
    record = Utils.ioResult(reader1.readNext());
    while (null != record) {
        DLMTestUtil.verifyEmptyLogRecord(record);
        ++numTxns;
        ++startTxID;
        assertEquals(startTxID, record.getTransactionId());
        assertEquals(record.getTransactionId() - 1L, record.getSequenceId());
        if (startTxID == txid - 1) {
            break;
        }
        record = Utils.ioResult(reader1.readNext());
    }
    assertEquals(txid - 1, startTxID);
    assertEquals(20, numTxns);
    Utils.close(reader1);
    dlm.close();
}

Example 3 with SubscriptionsStore

use of org.apache.distributedlog.api.subscription.SubscriptionsStore in project bookkeeper by apache.

the class BKDistributedLogManager method getAsyncLogReaderWithLock.

protected CompletableFuture<AsyncLogReader> getAsyncLogReaderWithLock(final Optional<DLSN> fromDLSN, final Optional<String> subscriberId) {
    if (!fromDLSN.isPresent() && !subscriberId.isPresent()) {
        return FutureUtils.exception(new UnexpectedException("Neither from dlsn nor subscriber id is provided."));
    }
    final BKAsyncLogReader reader = new BKAsyncLogReader(BKDistributedLogManager.this, scheduler, fromDLSN.isPresent() ? fromDLSN.get() : DLSN.InitialDLSN, subscriberId, false, statsLogger);
    pendingReaders.add(reader);
    final CompletableFuture<Void> lockFuture = reader.lockStream();
    final CompletableFuture<AsyncLogReader> createPromise = FutureUtils.createFuture();
    createPromise.whenComplete((value, cause) -> {
        if (cause instanceof CancellationException) {
            // cancel the lock when the creation future is cancelled
            lockFuture.cancel(true);
        }
    });
    // lock the stream - fetch the last commit position on success
    lockFuture.thenCompose(new Function<Void, CompletableFuture<AsyncLogReader>>() {

        @Override
        public CompletableFuture<AsyncLogReader> apply(Void complete) {
            if (fromDLSN.isPresent()) {
                return FutureUtils.value(reader);
            }
            LOG.info("Reader {} @ {} reading last commit position from subscription store after acquired lock.", subscriberId.get(), name);
            // we acquired lock
            final SubscriptionsStore subscriptionsStore = driver.getSubscriptionsStore(getStreamName());
            return subscriptionsStore.getLastCommitPosition(subscriberId.get()).thenCompose(lastCommitPosition -> {
                LOG.info("Reader {} @ {} positioned to last commit position {}.", new Object[] { subscriberId.get(), name, lastCommitPosition });
                try {
                    reader.setStartDLSN(lastCommitPosition);
                } catch (UnexpectedException e) {
                    return FutureUtils.exception(e);
                }
                return FutureUtils.value(reader);
            });
        }
    }).whenComplete(new FutureEventListener<AsyncLogReader>() {

        @Override
        public void onSuccess(AsyncLogReader r) {
            pendingReaders.remove(reader);
            FutureUtils.complete(createPromise, r);
        }

        @Override
        public void onFailure(final Throwable cause) {
            pendingReaders.remove(reader);
            FutureUtils.ensure(reader.asyncClose(), () -> FutureUtils.completeExceptionally(createPromise, cause));
        }
    });
    return createPromise;
}