Examples with DLSN org.apache.distributedlog.DLSN used on opensource projects

Example 6 with DLSN

use of org.apache.distributedlog.DLSN in project bookkeeper by apache.

the class TestLogRecordSelectors method testLastRecordSelector.

@Test(timeout = 60000)
public void testLastRecordSelector() {
    LastRecordSelector selector = new LastRecordSelector();
    for (int i = 0; i < 10; i++) {
        selector.process(DLMTestUtil.getLogRecordWithDLSNInstance(new DLSN(1L, i, 0L), i));
    }
    assertEquals(new DLSN(1L, 9L, 0L), selector.result().getDlsn());
}

Example 7 with DLSN

use of org.apache.distributedlog.DLSN in project bookkeeper by apache.

the class TestLogRecordSelectors method testFirstRecordSelector.

@Test(timeout = 60000)
public void testFirstRecordSelector() {
    FirstRecordSelector selectorIncludeControlRecord = new FirstRecordSelector(true);
    for (int i = 0; i < 5; i++) {
        selectorIncludeControlRecord.process(DLMTestUtil.getLogRecordWithDLSNInstance(new DLSN(1L, i * 2, 0L), i * 2, true));
        selectorIncludeControlRecord.process(DLMTestUtil.getLogRecordWithDLSNInstance(new DLSN(1L, i * 2 + 1, 0L), i * 2 + 1));
    }
    assertEquals(new DLSN(1L, 0L, 0L), selectorIncludeControlRecord.result().getDlsn());
    FirstRecordSelector selectorExcludeControlRecord = new FirstRecordSelector(false);
    for (int i = 0; i < 5; i++) {
        selectorExcludeControlRecord.process(DLMTestUtil.getLogRecordWithDLSNInstance(new DLSN(1L, i * 2, 0L), i * 2, true));
        selectorExcludeControlRecord.process(DLMTestUtil.getLogRecordWithDLSNInstance(new DLSN(1L, i * 2 + 1, 0L), i * 2 + 1));
    }
    assertEquals(new DLSN(1L, 1L, 0L), selectorExcludeControlRecord.result().getDlsn());
}

Example 8 with DLSN

use of org.apache.distributedlog.DLSN in project bookkeeper by apache.

the class TestDLCK method testCheckAndRepairDLNamespace.

@Test(timeout = 60000)
@SuppressWarnings("deprecation")
public void testCheckAndRepairDLNamespace() throws Exception {
    DistributedLogConfiguration confLocal = new DistributedLogConfiguration();
    confLocal.loadConf(conf);
    confLocal.setImmediateFlushEnabled(true);
    confLocal.setOutputBufferSize(0);
    confLocal.setLogSegmentSequenceNumberValidationEnabled(false);
    confLocal.setLogSegmentCacheEnabled(false);
    URI uri = createDLMURI("/check-and-repair-dl-namespace");
    zkc.get().create(uri.getPath(), new byte[0], ZooDefs.Ids.OPEN_ACL_UNSAFE, CreateMode.PERSISTENT);
    Namespace namespace = NamespaceBuilder.newBuilder().conf(confLocal).uri(uri).build();
    OrderedScheduler scheduler = OrderedScheduler.newSchedulerBuilder().name("dlck-tool").numThreads(1).build();
    ExecutorService executorService = Executors.newCachedThreadPool();
    String streamName = "check-and-repair-dl-namespace";
    // Create completed log segments
    DistributedLogManager dlm = namespace.openLog(streamName);
    DLMTestUtil.injectLogSegmentWithLastDLSN(dlm, confLocal, 1L, 1L, 10, false);
    DLMTestUtil.injectLogSegmentWithLastDLSN(dlm, confLocal, 2L, 11L, 10, true);
    DLMTestUtil.injectLogSegmentWithLastDLSN(dlm, confLocal, 3L, 21L, 10, false);
    DLMTestUtil.injectLogSegmentWithLastDLSN(dlm, confLocal, 4L, 31L, 10, true);
    // dryrun
    DistributedLogAdmin.checkAndRepairDLNamespace(uri, namespace, new DryrunLogSegmentMetadataStoreUpdater(confLocal, getLogSegmentMetadataStore(namespace)), scheduler, false, false);
    Map<Long, LogSegmentMetadata> segments = getLogSegments(dlm);
    LOG.info("segments after drynrun {}", segments);
    verifyLogSegment(segments, new DLSN(1L, 18L, 0L), 1L, 10, 10L);
    verifyLogSegment(segments, new DLSN(2L, 16L, 0L), 2L, 9, 19L);
    verifyLogSegment(segments, new DLSN(3L, 18L, 0L), 3L, 10, 30L);
    verifyLogSegment(segments, new DLSN(4L, 16L, 0L), 4L, 9, 39L);
    // check and repair
    DistributedLogAdmin.checkAndRepairDLNamespace(uri, namespace, LogSegmentMetadataStoreUpdater.createMetadataUpdater(confLocal, getLogSegmentMetadataStore(namespace)), scheduler, false, false);
    segments = getLogSegments(dlm);
    LOG.info("segments after repair {}", segments);
    verifyLogSegment(segments, new DLSN(1L, 18L, 0L), 1L, 10, 10L);
    verifyLogSegment(segments, new DLSN(2L, 18L, 0L), 2L, 10, 20L);
    verifyLogSegment(segments, new DLSN(3L, 18L, 0L), 3L, 10, 30L);
    verifyLogSegment(segments, new DLSN(4L, 18L, 0L), 4L, 10, 40L);
    dlm.close();
    SchedulerUtils.shutdownScheduler(executorService, 5, TimeUnit.MINUTES);
    namespace.close();
}

Example 9 with DLSN

use of org.apache.distributedlog.DLSN in project bookkeeper by apache.

the class TestDistributedLogAdmin method testChangeSequenceNumber.

/**
 * {@link https://issues.apache.org/jira/browse/DL-44}.
 */
@DistributedLogAnnotations.FlakyTest
@Ignore
@Test(timeout = 60000)
@SuppressWarnings("deprecation")
public void testChangeSequenceNumber() throws Exception {
    DistributedLogConfiguration confLocal = new DistributedLogConfiguration();
    confLocal.addConfiguration(conf);
    confLocal.setLogSegmentSequenceNumberValidationEnabled(false);
    confLocal.setLogSegmentCacheEnabled(false);
    DistributedLogConfiguration readConf = new DistributedLogConfiguration();
    readConf.addConfiguration(conf);
    readConf.setLogSegmentCacheEnabled(false);
    readConf.setLogSegmentSequenceNumberValidationEnabled(true);
    URI uri = createDLMURI("/change-sequence-number");
    zooKeeperClient.get().create(uri.getPath(), new byte[0], ZooDefs.Ids.OPEN_ACL_UNSAFE, CreateMode.PERSISTENT);
    Namespace namespace = NamespaceBuilder.newBuilder().conf(confLocal).uri(uri).build();
    Namespace readNamespace = NamespaceBuilder.newBuilder().conf(readConf).uri(uri).build();
    String streamName = "change-sequence-number";
    // create completed log segments
    DistributedLogManager dlm = namespace.openLog(streamName);
    DLMTestUtil.generateCompletedLogSegments(dlm, confLocal, 4, 10);
    DLMTestUtil.injectLogSegmentWithGivenLogSegmentSeqNo(dlm, confLocal, 5, 41, false, 10, true);
    dlm.close();
    // create a reader
    DistributedLogManager readDLM = readNamespace.openLog(streamName);
    AsyncLogReader reader = readDLM.getAsyncLogReader(DLSN.InitialDLSN);
    // read the records
    long expectedTxId = 1L;
    DLSN lastDLSN = DLSN.InitialDLSN;
    for (int i = 0; i < 4 * 10; i++) {
        LogRecordWithDLSN record = Utils.ioResult(reader.readNext());
        assertNotNull(record);
        DLMTestUtil.verifyLogRecord(record);
        assertEquals(expectedTxId, record.getTransactionId());
        expectedTxId++;
        lastDLSN = record.getDlsn();
    }
    LOG.info("Injecting bad log segment '3'");
    dlm = namespace.openLog(streamName);
    DLMTestUtil.injectLogSegmentWithGivenLogSegmentSeqNo(dlm, confLocal, 3L, 5 * 10 + 1, true, 10, false);
    LOG.info("Injected bad log segment '3'");
    // there isn't records should be read
    CompletableFuture<LogRecordWithDLSN> readFuture = reader.readNext();
    try {
        LogRecordWithDLSN record = Utils.ioResult(readFuture);
        fail("Should fail reading next record " + record + " when there is a corrupted log segment");
    } catch (UnexpectedException ue) {
    // expected
    }
    LOG.info("Dryrun fix inprogress segment that has lower sequence number");
    // Dryrun
    DistributedLogAdmin.fixInprogressSegmentWithLowerSequenceNumber(namespace, new DryrunLogSegmentMetadataStoreUpdater(confLocal, getLogSegmentMetadataStore(namespace)), streamName, false, false);
    try {
        reader = readDLM.getAsyncLogReader(lastDLSN);
        Utils.ioResult(reader.readNext());
        fail("Should fail reading next when there is a corrupted log segment");
    } catch (UnexpectedException ue) {
    // expected
    }
    LOG.info("Actual run fix inprogress segment that has lower sequence number");
    // Actual run
    DistributedLogAdmin.fixInprogressSegmentWithLowerSequenceNumber(namespace, LogSegmentMetadataStoreUpdater.createMetadataUpdater(confLocal, getLogSegmentMetadataStore(namespace)), streamName, false, false);
    // be able to read more after fix
    reader = readDLM.getAsyncLogReader(lastDLSN);
    // skip the first record
    Utils.ioResult(reader.readNext());
    readFuture = reader.readNext();
    expectedTxId = 51L;
    LogRecord record = Utils.ioResult(readFuture);
    assertNotNull(record);
    DLMTestUtil.verifyLogRecord(record);
    assertEquals(expectedTxId, record.getTransactionId());
    expectedTxId++;
    for (int i = 1; i < 10; i++) {
        record = Utils.ioResult(reader.readNext());
        assertNotNull(record);
        DLMTestUtil.verifyLogRecord(record);
        assertEquals(expectedTxId, record.getTransactionId());
        expectedTxId++;
    }
    Utils.close(reader);
    readDLM.close();
    dlm.close();
    namespace.close();
    readNamespace.close();
}

Example 10 with DLSN

use of org.apache.distributedlog.DLSN in project bookkeeper by apache.

the class AbstractStateStoreWithJournal method getLastDLSN.

private CompletableFuture<DLSN> getLastDLSN(StateStoreSpec spec) {
    synchronized (this) {
        if (null != closeFuture) {
            return FutureUtils.exception(new StateStoreClosedException(name()));
        }
    }
    try {
        logManager = logNamespace.openLog(spec.getStream());
    } catch (IOException e) {
        return FutureUtils.exception(e);
    }
    CompletableFuture<DLSN> future = FutureUtils.createFuture();
    logManager.getLastDLSNAsync().whenCompleteAsync(new FutureEventListener<DLSN>() {

        @Override
        public void onSuccess(DLSN dlsn) {
            future.complete(dlsn);
        }

        @Override
        public void onFailure(Throwable throwable) {
            if (throwable instanceof LogEmptyException || throwable instanceof LogNotFoundException) {
                FutureUtils.proxyTo(writeCatchUpMarker(), future);
            } else {
                future.completeExceptionally(throwable);
            }
        }
    });
    return future;
}