Examples with LogSegmentMetadata org.apache.distributedlog.LogSegmentMetadata used on opensource projects

Example 11 with LogSegmentMetadata

use of org.apache.distributedlog.LogSegmentMetadata in project bookkeeper by apache.

the class TestBKLogSegmentEntryReader method testMaxPrefetchEntriesLargeBatch.

@Test(timeout = 60000)
public void testMaxPrefetchEntriesLargeBatch() throws Exception {
    DistributedLogConfiguration confLocal = new DistributedLogConfiguration();
    confLocal.addConfiguration(conf);
    confLocal.setOutputBufferSize(0);
    confLocal.setPeriodicFlushFrequencyMilliSeconds(0);
    confLocal.setImmediateFlushEnabled(false);
    confLocal.setNumPrefetchEntriesPerLogSegment(10);
    confLocal.setMaxPrefetchEntriesPerLogSegment(5);
    DistributedLogManager dlm = createNewDLM(confLocal, runtime.getMethodName());
    generateCompletedLogSegments(dlm, confLocal, 1, 20);
    List<LogSegmentMetadata> segments = dlm.getLogSegments();
    assertEquals(segments.size() + " log segments found, expected to be only one", 1, segments.size());
    BKLogSegmentEntryReader reader = createEntryReader(segments.get(0), 0, confLocal);
    reader.start();
    // wait for the read ahead entries to become available
    while (reader.readAheadEntries.size() < 5) {
        TimeUnit.MILLISECONDS.sleep(10);
    }
    long txId = 1L;
    long entryId = 0L;
    assertEquals(5, reader.readAheadEntries.size());
    assertEquals(5, reader.getNextEntryId());
    // read first entry
    Entry.Reader entryReader = Utils.ioResult(reader.readNext(1)).get(0);
    LogRecordWithDLSN record = entryReader.nextRecord();
    while (null != record) {
        if (!record.isControl()) {
            DLMTestUtil.verifyLogRecord(record);
            assertEquals(txId, record.getTransactionId());
            ++txId;
        }
        DLSN dlsn = record.getDlsn();
        assertEquals(1L, dlsn.getLogSegmentSequenceNo());
        assertEquals(entryId, dlsn.getEntryId());
        record = entryReader.nextRecord();
    }
    ++entryId;
    assertEquals(2L, txId);
    // wait for the read ahead entries to become 10 again
    while (reader.readAheadEntries.size() < 5) {
        TimeUnit.MILLISECONDS.sleep(10);
    }
    assertEquals(5, reader.readAheadEntries.size());
    assertEquals(6, reader.getNextEntryId());
    assertFalse(reader.hasCaughtUpOnInprogress());
    Utils.close(reader);
}

Example 12 with LogSegmentMetadata

use of org.apache.distributedlog.LogSegmentMetadata in project bookkeeper by apache.

the class TestZKLogStreamMetadataStore method createLog.

private static void createLog(ZooKeeperClient zk, URI uri, String logName, String logIdentifier, int numSegments) throws Exception {
    final String logRootPath = getLogRootPath(uri, logName, logIdentifier);
    final String logSegmentsPath = logRootPath + LOGSEGMENTS_PATH;
    final String maxTxIdPath = logRootPath + MAX_TXID_PATH;
    final String lockPath = logRootPath + LOCK_PATH;
    final String readLockPath = logRootPath + READ_LOCK_PATH;
    final String versionPath = logRootPath + VERSION_PATH;
    final String allocationPath = logRootPath + ALLOCATION_PATH;
    Utils.zkCreateFullPathOptimistic(zk, logRootPath, new byte[0], zk.getDefaultACL(), CreateMode.PERSISTENT);
    Transaction txn = zk.get().transaction();
    txn.create(logSegmentsPath, DLUtils.serializeLogSegmentSequenceNumber(DistributedLogConstants.UNASSIGNED_LOGSEGMENT_SEQNO), zk.getDefaultACL(), CreateMode.PERSISTENT);
    txn.create(maxTxIdPath, DLUtils.serializeTransactionId(0L), zk.getDefaultACL(), CreateMode.PERSISTENT);
    txn.create(lockPath, EMPTY_BYTES, zk.getDefaultACL(), CreateMode.PERSISTENT);
    txn.create(readLockPath, EMPTY_BYTES, zk.getDefaultACL(), CreateMode.PERSISTENT);
    txn.create(versionPath, intToBytes(LAYOUT_VERSION), zk.getDefaultACL(), CreateMode.PERSISTENT);
    txn.create(allocationPath, EMPTY_BYTES, zk.getDefaultACL(), CreateMode.PERSISTENT);
    for (int i = 0; i < numSegments; i++) {
        LogSegmentMetadata segment = DLMTestUtil.completedLogSegment(logSegmentsPath, i + 1L, 1L + i * 1000L, (i + 1) * 1000L, 1000, i + 1L, 999L, 0L);
        txn.create(segment.getZkPath(), segment.getFinalisedData().getBytes(UTF_8), zk.getDefaultACL(), CreateMode.PERSISTENT);
    }
    txn.commit();
}

Example 13 with LogSegmentMetadata

use of org.apache.distributedlog.LogSegmentMetadata in project bookkeeper by apache.

the class TestPerStreamLogSegmentCache method testGapDetectionOnLogSegmentsWithoutLogSegmentSequenceNumber.

@Test(timeout = 60000)
public void testGapDetectionOnLogSegmentsWithoutLogSegmentSequenceNumber() throws Exception {
    PerStreamLogSegmentCache cache = new PerStreamLogSegmentCache("test-gap-detection");
    LogSegmentMetadata segment1 = DLMTestUtil.completedLogSegment("/segment-1", 1L, 1L, 100L, 100, 1L, 99L, 0L).mutator().setVersion(LogSegmentMetadata.LogSegmentMetadataVersion.VERSION_V1_ORIGINAL).build();
    cache.add(DLMTestUtil.completedLedgerZNodeNameWithLogSegmentSequenceNumber(1L), segment1);
    LogSegmentMetadata segment3 = DLMTestUtil.completedLogSegment("/segment-3", 3L, 3L, 300L, 100, 3L, 99L, 0L).mutator().setVersion(LogSegmentMetadata.LogSegmentMetadataVersion.VERSION_V2_LEDGER_SEQNO).build();
    cache.add(DLMTestUtil.completedLedgerZNodeNameWithLogSegmentSequenceNumber(3L), segment3);
    List<LogSegmentMetadata> expectedList = Lists.asList(segment1, new LogSegmentMetadata[] { segment3 });
    List<LogSegmentMetadata> resultList = cache.getLogSegments(LogSegmentMetadata.COMPARATOR);
    assertEquals(expectedList, resultList);
}

Example 14 with LogSegmentMetadata

use of org.apache.distributedlog.LogSegmentMetadata in project bookkeeper by apache.

the class TestPerStreamLogSegmentCache method testDiff.

@Test(timeout = 60000)
public void testDiff() {
    PerStreamLogSegmentCache cache = new PerStreamLogSegmentCache("test-diff");
    // add 5 completed log segments
    for (int i = 1; i <= 5; i++) {
        LogSegmentMetadata metadata = DLMTestUtil.completedLogSegment("/segment" + i, i, i, i * 100L, 100, i, 99L, 0L);
        String name = DLMTestUtil.completedLedgerZNodeNameWithLogSegmentSequenceNumber(i);
        cache.add(name, metadata);
    }
    // add one inprogress log segment
    LogSegmentMetadata inprogress = DLMTestUtil.inprogressLogSegment("/inprogress-6", 6, 600L, 6);
    String name = DLMTestUtil.inprogressZNodeName(6);
    cache.add(name, inprogress);
    // deleted first 2 completed log segments and completed the last one
    Set<String> segmentRemoved = Sets.newHashSet();
    for (int i = 1; i <= 2; i++) {
        segmentRemoved.add(DLMTestUtil.completedLedgerZNodeNameWithLogSegmentSequenceNumber(i));
    }
    segmentRemoved.add((DLMTestUtil.inprogressZNodeName(6)));
    Set<String> segmentReceived = Sets.newHashSet();
    Set<String> segmentAdded = Sets.newHashSet();
    for (int i = 3; i <= 6; i++) {
        segmentReceived.add(DLMTestUtil.completedLedgerZNodeNameWithLogSegmentSequenceNumber(i));
        if (i == 6) {
            segmentAdded.add(DLMTestUtil.completedLedgerZNodeNameWithLogSegmentSequenceNumber(i));
        }
    }
    Pair<Set<String>, Set<String>> segmentChanges = cache.diff(segmentReceived);
    assertTrue("Should remove " + segmentRemoved + ", but removed " + segmentChanges.getRight(), Sets.difference(segmentRemoved, segmentChanges.getRight()).isEmpty());
    assertTrue("Should add " + segmentAdded + ", but added " + segmentChanges.getLeft(), Sets.difference(segmentAdded, segmentChanges.getLeft()).isEmpty());
}

Example 15 with LogSegmentMetadata

use of org.apache.distributedlog.LogSegmentMetadata in project bookkeeper by apache.

the class TestLogSegmentMetadataStoreUpdater method testUpdateLastDLSN.

@Test(timeout = 60000)
public void testUpdateLastDLSN() throws Exception {
    String ledgerPath = "/testUpdateLastDLSN";
    zkc.get().create(ledgerPath, new byte[0], ZooDefs.Ids.OPEN_ACL_UNSAFE, CreateMode.PERSISTENT);
    // Create 1 completed log segment
    LogSegmentMetadata completedLogSegment = DLMTestUtil.completedLogSegment(ledgerPath, 1L, 0L, 99L, 100, 1L, 99L, 0L);
    completedLogSegment.write(zkc);
    // Create 1 inprogress log segment
    LogSegmentMetadata inprogressLogSegment = DLMTestUtil.inprogressLogSegment(ledgerPath, 2L, 100L, 2L);
    inprogressLogSegment.write(zkc);
    DLSN badLastDLSN = new DLSN(99L, 0L, 0L);
    DLSN goodLastDLSN1 = new DLSN(1L, 100L, 0L);
    DLSN goodLastDLSN2 = new DLSN(2L, 200L, 0L);
    LogRecordWithDLSN badRecord = DLMTestUtil.getLogRecordWithDLSNInstance(badLastDLSN, 100L);
    LogRecordWithDLSN goodRecord1 = DLMTestUtil.getLogRecordWithDLSNInstance(goodLastDLSN1, 100L);
    LogRecordWithDLSN goodRecord2 = DLMTestUtil.getLogRecordWithDLSNInstance(goodLastDLSN2, 200L);
    // Dryrun
    MetadataUpdater dryrunUpdater = new DryrunLogSegmentMetadataStoreUpdater(conf, metadataStore);
    try {
        Utils.ioResult(dryrunUpdater.updateLastRecord(completedLogSegment, badRecord));
        fail("Should fail on updating dlsn that in different log segment");
    } catch (IllegalArgumentException iae) {
    // expected
    }
    try {
        Utils.ioResult(dryrunUpdater.updateLastRecord(inprogressLogSegment, goodRecord2));
        fail("Should fail on updating dlsn for an inprogress log segment");
    } catch (IllegalStateException ise) {
    // expected
    }
    LogSegmentMetadata updatedCompletedLogSegment = Utils.ioResult(dryrunUpdater.updateLastRecord(completedLogSegment, goodRecord1));
    assertEquals(goodLastDLSN1, updatedCompletedLogSegment.getLastDLSN());
    assertEquals(goodRecord1.getTransactionId(), updatedCompletedLogSegment.getLastTxId());
    assertTrue(updatedCompletedLogSegment.isRecordLastPositioninThisSegment(goodRecord1));
    Map<Long, LogSegmentMetadata> segmentList = readLogSegments(ledgerPath);
    assertEquals(2, segmentList.size());
    LogSegmentMetadata readCompletedLogSegment = segmentList.get(1L);
    assertNotNull(readCompletedLogSegment);
    assertEquals(completedLogSegment, readCompletedLogSegment);
    LogSegmentMetadata readInprogressLogSegment = segmentList.get(2L);
    assertNotNull(readInprogressLogSegment);
    assertEquals(inprogressLogSegment, readInprogressLogSegment);
    // Fix the last dlsn
    MetadataUpdater updater = LogSegmentMetadataStoreUpdater.createMetadataUpdater(conf, metadataStore);
    try {
        Utils.ioResult(updater.updateLastRecord(completedLogSegment, badRecord));
        fail("Should fail on updating dlsn that in different log segment");
    } catch (IllegalArgumentException iae) {
    // expected
    }
    try {
        Utils.ioResult(updater.updateLastRecord(inprogressLogSegment, goodRecord2));
        fail("Should fail on updating dlsn for an inprogress log segment");
    } catch (IllegalStateException ise) {
    // expected
    }
    updatedCompletedLogSegment = Utils.ioResult(updater.updateLastRecord(completedLogSegment, goodRecord1));
    assertEquals(goodLastDLSN1, updatedCompletedLogSegment.getLastDLSN());
    assertEquals(goodRecord1.getTransactionId(), updatedCompletedLogSegment.getLastTxId());
    assertTrue(updatedCompletedLogSegment.isRecordLastPositioninThisSegment(goodRecord1));
    segmentList = readLogSegments(ledgerPath);
    assertEquals(2, segmentList.size());
    readCompletedLogSegment = segmentList.get(1L);
    assertNotNull(readCompletedLogSegment);
    assertEquals(goodLastDLSN1, readCompletedLogSegment.getLastDLSN());
    assertEquals(goodRecord1.getTransactionId(), readCompletedLogSegment.getLastTxId());
    assertTrue(readCompletedLogSegment.isRecordLastPositioninThisSegment(goodRecord1));
    assertEquals(updatedCompletedLogSegment, readCompletedLogSegment);
    assertEquals(completedLogSegment.getCompletionTime(), readCompletedLogSegment.getCompletionTime());
    assertEquals(completedLogSegment.getFirstTxId(), readCompletedLogSegment.getFirstTxId());
    assertEquals(completedLogSegment.getLogSegmentId(), readCompletedLogSegment.getLogSegmentId());
    assertEquals(completedLogSegment.getLogSegmentSequenceNumber(), readCompletedLogSegment.getLogSegmentSequenceNumber());
    assertEquals(completedLogSegment.getRegionId(), readCompletedLogSegment.getRegionId());
    assertEquals(completedLogSegment.getZkPath(), readCompletedLogSegment.getZkPath());
    assertEquals(completedLogSegment.getZNodeName(), readCompletedLogSegment.getZNodeName());
    readInprogressLogSegment = segmentList.get(2L);
    assertNotNull(readInprogressLogSegment);
    assertEquals(inprogressLogSegment, readInprogressLogSegment);
}