Examples with AsyncLogReader org.apache.distributedlog.api.AsyncLogReader used on opensource projects

Example 31 with AsyncLogReader

use of org.apache.distributedlog.api.AsyncLogReader in project bookkeeper by apache.

the class TestBKDistributedLogManager method testTruncationValidation.

@Test(timeout = 60000)
public void testTruncationValidation() throws Exception {
    String name = "distrlog-truncation-validation";
    URI uri = createDLMURI("/" + name);
    ZooKeeperClient zookeeperClient = TestZooKeeperClientBuilder.newBuilder().uri(uri).build();
    OrderedScheduler scheduler = OrderedScheduler.newSchedulerBuilder().name("test-truncation-validation").numThreads(1).build();
    DistributedLogConfiguration confLocal = new DistributedLogConfiguration();
    confLocal.loadConf(conf);
    confLocal.setDLLedgerMetadataLayoutVersion(LogSegmentMetadata.LEDGER_METADATA_CURRENT_LAYOUT_VERSION);
    confLocal.setOutputBufferSize(0);
    confLocal.setLogSegmentCacheEnabled(false);
    LogSegmentMetadataStore metadataStore = new ZKLogSegmentMetadataStore(confLocal, zookeeperClient, scheduler);
    BKDistributedLogManager dlm = createNewDLM(confLocal, name);
    DLSN truncDLSN = DLSN.InitialDLSN;
    DLSN beyondTruncDLSN = DLSN.InitialDLSN;
    long beyondTruncTxId = 1;
    long txid = 1;
    for (long i = 0; i < 3; i++) {
        long start = txid;
        BKAsyncLogWriter writer = dlm.startAsyncLogSegmentNonPartitioned();
        for (long j = 1; j <= 10; j++) {
            LogRecord record = DLMTestUtil.getLargeLogRecordInstance(txid++);
            CompletableFuture<DLSN> dlsn = writer.write(record);
            if (i == 1 && j == 2) {
                truncDLSN = Utils.ioResult(dlsn);
            } else if (i == 2 && j == 3) {
                beyondTruncDLSN = Utils.ioResult(dlsn);
                beyondTruncTxId = record.getTransactionId();
            } else if (j == 10) {
                Utils.ioResult(dlsn);
            }
        }
        writer.close();
    }
    {
        LogReader reader = dlm.getInputStream(DLSN.InitialDLSN);
        LogRecordWithDLSN record = reader.readNext(false);
        assertTrue((record != null) && (record.getDlsn().compareTo(DLSN.InitialDLSN) == 0));
        reader.close();
    }
    Map<Long, LogSegmentMetadata> segmentList = DLMTestUtil.readLogSegments(zookeeperClient, LogMetadata.getLogSegmentsPath(uri, name, confLocal.getUnpartitionedStreamName()));
    LOG.info("Read segments before truncating first segment : {}", segmentList);
    MetadataUpdater updater = LogSegmentMetadataStoreUpdater.createMetadataUpdater(confLocal, metadataStore);
    Utils.ioResult(updater.setLogSegmentTruncated(segmentList.get(1L)));
    segmentList = DLMTestUtil.readLogSegments(zookeeperClient, LogMetadata.getLogSegmentsPath(uri, name, confLocal.getUnpartitionedStreamName()));
    LOG.info("Read segments after truncated first segment : {}", segmentList);
    {
        LogReader reader = dlm.getInputStream(DLSN.InitialDLSN);
        LogRecordWithDLSN record = reader.readNext(false);
        assertTrue("Unexpected record : " + record, (record != null) && (record.getDlsn().compareTo(new DLSN(2, 0, 0)) == 0));
        reader.close();
    }
    {
        LogReader reader = dlm.getInputStream(1);
        LogRecordWithDLSN record = reader.readNext(false);
        assertTrue((record != null) && (record.getDlsn().compareTo(new DLSN(2, 0, 0)) == 0));
        reader.close();
    }
    updater = LogSegmentMetadataStoreUpdater.createMetadataUpdater(confLocal, metadataStore);
    Utils.ioResult(updater.setLogSegmentActive(segmentList.get(1L)));
    segmentList = DLMTestUtil.readLogSegments(zookeeperClient, LogMetadata.getLogSegmentsPath(uri, name, confLocal.getUnpartitionedStreamName()));
    LOG.info("Read segments after marked first segment as active : {}", segmentList);
    updater = LogSegmentMetadataStoreUpdater.createMetadataUpdater(confLocal, metadataStore);
    Utils.ioResult(updater.setLogSegmentTruncated(segmentList.get(2L)));
    segmentList = DLMTestUtil.readLogSegments(zookeeperClient, LogMetadata.getLogSegmentsPath(uri, name, confLocal.getUnpartitionedStreamName()));
    LOG.info("Read segments after truncated second segment : {}", segmentList);
    {
        AsyncLogReader reader = dlm.getAsyncLogReader(DLSN.InitialDLSN);
        long expectedTxId = 1L;
        boolean exceptionEncountered = false;
        try {
            for (int i = 0; i < 3 * 10; i++) {
                LogRecordWithDLSN record = Utils.ioResult(reader.readNext());
                DLMTestUtil.verifyLargeLogRecord(record);
                assertEquals(expectedTxId, record.getTransactionId());
                expectedTxId++;
            }
        } catch (AlreadyTruncatedTransactionException exc) {
            exceptionEncountered = true;
        }
        assertTrue(exceptionEncountered);
        Utils.close(reader);
    }
    updater = LogSegmentMetadataStoreUpdater.createMetadataUpdater(conf, metadataStore);
    Utils.ioResult(updater.setLogSegmentActive(segmentList.get(2L)));
    BKAsyncLogWriter writer = dlm.startAsyncLogSegmentNonPartitioned();
    assertTrue(Utils.ioResult(writer.truncate(truncDLSN)));
    BKLogWriteHandler handler = writer.getCachedWriteHandler();
    List<LogSegmentMetadata> cachedSegments = handler.getCachedLogSegments(LogSegmentMetadata.COMPARATOR);
    for (LogSegmentMetadata segment : cachedSegments) {
        if (segment.getLastDLSN().compareTo(truncDLSN) < 0) {
            assertTrue(segment.isTruncated());
            assertTrue(!segment.isPartiallyTruncated());
        } else if (segment.getFirstDLSN().compareTo(truncDLSN) < 0) {
            assertTrue(!segment.isTruncated());
            assertTrue(segment.isPartiallyTruncated());
        } else {
            assertTrue(!segment.isTruncated());
            assertTrue(!segment.isPartiallyTruncated());
        }
    }
    segmentList = DLMTestUtil.readLogSegments(zookeeperClient, LogMetadata.getLogSegmentsPath(uri, name, conf.getUnpartitionedStreamName()));
    assertTrue(segmentList.get(truncDLSN.getLogSegmentSequenceNo()).getMinActiveDLSN().compareTo(truncDLSN) == 0);
    {
        LogReader reader = dlm.getInputStream(DLSN.InitialDLSN);
        LogRecordWithDLSN record = reader.readNext(false);
        assertTrue(record != null);
        assertEquals(truncDLSN, record.getDlsn());
        reader.close();
    }
    {
        LogReader reader = dlm.getInputStream(1);
        LogRecordWithDLSN record = reader.readNext(false);
        assertTrue(record != null);
        assertEquals(truncDLSN, record.getDlsn());
        reader.close();
    }
    {
        AsyncLogReader reader = dlm.getAsyncLogReader(DLSN.InitialDLSN);
        LogRecordWithDLSN record = Utils.ioResult(reader.readNext());
        assertTrue(record != null);
        assertEquals(truncDLSN, record.getDlsn());
        Utils.close(reader);
    }
    {
        LogReader reader = dlm.getInputStream(beyondTruncDLSN);
        LogRecordWithDLSN record = reader.readNext(false);
        assertTrue(record != null);
        assertEquals(beyondTruncDLSN, record.getDlsn());
        reader.close();
    }
    {
        LogReader reader = dlm.getInputStream(beyondTruncTxId);
        LogRecordWithDLSN record = reader.readNext(false);
        assertTrue(record != null);
        assertEquals(beyondTruncDLSN, record.getDlsn());
        assertEquals(beyondTruncTxId, record.getTransactionId());
        reader.close();
    }
    {
        AsyncLogReader reader = dlm.getAsyncLogReader(beyondTruncDLSN);
        LogRecordWithDLSN record = Utils.ioResult(reader.readNext());
        assertTrue(record != null);
        assertEquals(beyondTruncDLSN, record.getDlsn());
        Utils.close(reader);
    }
    zookeeperClient.close();
}

Example 32 with AsyncLogReader

use of org.apache.distributedlog.api.AsyncLogReader in project bookkeeper by apache.

the class BKDistributedLogManager method getAsyncLogReaderWithLock.

protected CompletableFuture<AsyncLogReader> getAsyncLogReaderWithLock(final Optional<DLSN> fromDLSN, final Optional<String> subscriberId) {
    if (!fromDLSN.isPresent() && !subscriberId.isPresent()) {
        return FutureUtils.exception(new UnexpectedException("Neither from dlsn nor subscriber id is provided."));
    }
    final BKAsyncLogReader reader = new BKAsyncLogReader(BKDistributedLogManager.this, scheduler, fromDLSN.isPresent() ? fromDLSN.get() : DLSN.InitialDLSN, subscriberId, false, statsLogger);
    pendingReaders.add(reader);
    final CompletableFuture<Void> lockFuture = reader.lockStream();
    final CompletableFuture<AsyncLogReader> createPromise = FutureUtils.createFuture();
    createPromise.whenComplete((value, cause) -> {
        if (cause instanceof CancellationException) {
            // cancel the lock when the creation future is cancelled
            lockFuture.cancel(true);
        }
    });
    // lock the stream - fetch the last commit position on success
    lockFuture.thenCompose(new Function<Void, CompletableFuture<AsyncLogReader>>() {

        @Override
        public CompletableFuture<AsyncLogReader> apply(Void complete) {
            if (fromDLSN.isPresent()) {
                return FutureUtils.value(reader);
            }
            LOG.info("Reader {} @ {} reading last commit position from subscription store after acquired lock.", subscriberId.get(), name);
            // we acquired lock
            final SubscriptionsStore subscriptionsStore = driver.getSubscriptionsStore(getStreamName());
            return subscriptionsStore.getLastCommitPosition(subscriberId.get()).thenCompose(lastCommitPosition -> {
                LOG.info("Reader {} @ {} positioned to last commit position {}.", new Object[] { subscriberId.get(), name, lastCommitPosition });
                try {
                    reader.setStartDLSN(lastCommitPosition);
                } catch (UnexpectedException e) {
                    return FutureUtils.exception(e);
                }
                return FutureUtils.value(reader);
            });
        }
    }).whenComplete(new FutureEventListener<AsyncLogReader>() {

        @Override
        public void onSuccess(AsyncLogReader r) {
            pendingReaders.remove(reader);
            FutureUtils.complete(createPromise, r);
        }

        @Override
        public void onFailure(final Throwable cause) {
            pendingReaders.remove(reader);
            FutureUtils.ensure(reader.asyncClose(), () -> FutureUtils.completeExceptionally(createPromise, cause));
        }
    });
    return createPromise;
}

Example 33 with AsyncLogReader

use of org.apache.distributedlog.api.AsyncLogReader in project bookkeeper by apache.

the class MVCCStoreFactoryImplTest method setup.

@Before
public void setup() throws IOException {
    this.namespace = mock(Namespace.class);
    DistributedLogManager dlm = mock(DistributedLogManager.class);
    when(dlm.asyncClose()).thenReturn(FutureUtils.Void());
    when(namespace.openLog(anyString())).thenReturn(dlm);
    AsyncLogWriter logWriter = mock(AsyncLogWriter.class);
    when(dlm.openAsyncLogWriter()).thenReturn(FutureUtils.value(logWriter));
    when(logWriter.getLastTxId()).thenReturn(-1L);
    DLSN dlsn = new DLSN(0L, 0L, 0L);
    when(logWriter.write(any(LogRecord.class))).thenReturn(FutureUtils.value(dlsn));
    when(logWriter.asyncClose()).thenReturn(FutureUtils.Void());
    AsyncLogReader logReader = mock(AsyncLogReader.class);
    when(dlm.openAsyncLogReader(anyLong())).thenReturn(FutureUtils.value(logReader));
    when(logReader.asyncClose()).thenReturn(FutureUtils.Void());
    LogRecordWithDLSN record = new LogRecordWithDLSN(dlsn, 0L, NOP_CMD.toByteArray(), 0L);
    when(logReader.readNext()).thenReturn(FutureUtils.value(record));
    int numDirs = 3;
    this.storeDirs = new File[numDirs];
    for (int i = 0; i < numDirs; i++) {
        storeDirs[i] = testDir.newFolder("test-" + i);
    }
    this.resources = StorageResources.create(StorageResourcesSpec.builder().numCheckpointThreads(3).numIOReadThreads(3).numIOWriteThreads(3).build());
    this.factory = new MVCCStoreFactoryImpl(() -> namespace, storeDirs, resources, false);
}

Example 34 with AsyncLogReader

use of org.apache.distributedlog.api.AsyncLogReader in project bookkeeper by apache.

the class MVCCAsyncStoreTestBase method mockNamespace.

private static Namespace mockNamespace() throws Exception {
    Namespace namespace = mock(Namespace.class);
    DistributedLogManager dlm = mock(DistributedLogManager.class);
    when(dlm.asyncClose()).thenReturn(FutureUtils.Void());
    when(namespace.openLog(anyString())).thenReturn(dlm);
    AsyncLogWriter logWriter = mock(AsyncLogWriter.class);
    when(dlm.openAsyncLogWriter()).thenReturn(FutureUtils.value(logWriter));
    when(logWriter.getLastTxId()).thenReturn(-1L);
    DLSN dlsn = new DLSN(0L, 0L, 0L);
    when(logWriter.write(any(LogRecord.class))).thenReturn(FutureUtils.value(dlsn));
    when(logWriter.asyncClose()).thenReturn(FutureUtils.Void());
    AsyncLogReader logReader = mock(AsyncLogReader.class);
    when(dlm.openAsyncLogReader(anyLong())).thenReturn(FutureUtils.value(logReader));
    when(logReader.asyncClose()).thenReturn(FutureUtils.Void());
    LogRecordWithDLSN record = new LogRecordWithDLSN(dlsn, 0L, NOP_CMD.toByteArray(), 0L);
    when(logReader.readNext()).thenReturn(FutureUtils.value(record));
    return namespace;
}