Examples with LogSegmentMetadata com.twitter.distributedlog.LogSegmentMetadata used on opensource projects

Example 21 with LogSegmentMetadata

use of com.twitter.distributedlog.LogSegmentMetadata in project distributedlog by twitter.

the class DLAuditor method calculateStreamSpaceUsage.

private long calculateStreamSpaceUsage(final com.twitter.distributedlog.DistributedLogManagerFactory factory, final String stream) throws IOException {
    DistributedLogManager dlm = factory.createDistributedLogManager(stream, com.twitter.distributedlog.DistributedLogManagerFactory.ClientSharingOption.SharedClients);
    long totalBytes = 0;
    try {
        List<LogSegmentMetadata> segments = dlm.getLogSegments();
        for (LogSegmentMetadata segment : segments) {
            try {
                LedgerHandle lh = getBookKeeperClient(factory).get().openLedgerNoRecovery(segment.getLedgerId(), BookKeeper.DigestType.CRC32, conf.getBKDigestPW().getBytes(UTF_8));
                totalBytes += lh.getLength();
                lh.close();
            } catch (BKException e) {
                logger.error("Failed to open ledger {} : ", segment.getLedgerId(), e);
                throw new IOException("Failed to open ledger " + segment.getLedgerId(), e);
            } catch (InterruptedException e) {
                logger.warn("Interrupted on opening ledger {} : ", segment.getLedgerId(), e);
                Thread.currentThread().interrupt();
                throw new DLInterruptedException("Interrupted on opening ledger " + segment.getLedgerId(), e);
            }
        }
    } finally {
        dlm.close();
    }
    return totalBytes;
}

Example 22 with LogSegmentMetadata

use of com.twitter.distributedlog.LogSegmentMetadata in project distributedlog by twitter.

the class DLUtils method nextLogSegmentSequenceNumber.

/**
     * Assign next log segment sequence number based on a decreasing list of log segments.
     *
     * @param segmentListDesc
     *          a decreasing list of log segments
     * @return null if no log segments was assigned a sequence number in <code>segmentListDesc</code>.
     *         otherwise, return next log segment sequence number
     */
public static Long nextLogSegmentSequenceNumber(List<LogSegmentMetadata> segmentListDesc) {
    int lastAssignedLogSegmentIdx = -1;
    Long lastAssignedLogSegmentSeqNo = null;
    Long nextLogSegmentSeqNo = null;
    for (int i = 0; i < segmentListDesc.size(); i++) {
        LogSegmentMetadata metadata = segmentListDesc.get(i);
        if (LogSegmentMetadata.supportsLogSegmentSequenceNo(metadata.getVersion())) {
            lastAssignedLogSegmentSeqNo = metadata.getLogSegmentSequenceNumber();
            lastAssignedLogSegmentIdx = i;
            break;
        }
    }
    if (null != lastAssignedLogSegmentSeqNo) {
        // latest log segment is assigned with a sequence number, start with next sequence number
        nextLogSegmentSeqNo = lastAssignedLogSegmentSeqNo + lastAssignedLogSegmentIdx + 1;
    }
    return nextLogSegmentSeqNo;
}

Example 23 with LogSegmentMetadata

use of com.twitter.distributedlog.LogSegmentMetadata in project distributedlog by twitter.

the class DLUtils method findLogSegmentNotLessThanTxnId.

/**
     * Find the log segment whose transaction ids are not less than provided <code>transactionId</code>.
     *
     * @param segments
     *          segments to search
     * @param transactionId
     *          transaction id to find
     * @return the first log segment whose transaction ids are not less than <code>transactionId</code>.
     */
public static int findLogSegmentNotLessThanTxnId(List<LogSegmentMetadata> segments, long transactionId) {
    int found = -1;
    for (int i = segments.size() - 1; i >= 0; i--) {
        LogSegmentMetadata segment = segments.get(i);
        if (segment.getFirstTxId() <= transactionId) {
            found = i;
            break;
        }
    }
    if (found <= -1) {
        return -1;
    }
    if (found == 0 && segments.get(0).getFirstTxId() == transactionId) {
        return 0;
    }
    LogSegmentMetadata foundSegment = segments.get(found);
    if (foundSegment.getFirstTxId() == transactionId) {
        for (int i = found - 1; i >= 0; i--) {
            LogSegmentMetadata segment = segments.get(i);
            if (segment.isInProgress()) {
                break;
            }
            if (segment.getLastTxId() < transactionId) {
                break;
            }
            found = i;
        }
        return found;
    } else {
        if (foundSegment.isInProgress() || found == segments.size() - 1) {
            return found;
        }
        if (foundSegment.getLastTxId() >= transactionId) {
            return found;
        }
        return found + 1;
    }
}

Example 24 with LogSegmentMetadata

use of com.twitter.distributedlog.LogSegmentMetadata in project distributedlog by twitter.

the class LogSegmentMetadataStoreUpdater method changeSequenceNumber.

@Override
public Future<LogSegmentMetadata> changeSequenceNumber(LogSegmentMetadata segment, long logSegmentSeqNo) {
    String newZkPath = segment.getZkPath().replace(formatLogSegmentSequenceNumber(segment.getLogSegmentSequenceNumber()), formatLogSegmentSequenceNumber(logSegmentSeqNo));
    final LogSegmentMetadata newSegment = segment.mutator().setLogSegmentSequenceNumber(logSegmentSeqNo).setZkPath(newZkPath).build();
    return addNewSegmentAndDeleteOldSegment(newSegment, segment);
}

Example 25 with LogSegmentMetadata

use of com.twitter.distributedlog.LogSegmentMetadata in project distributedlog by twitter.

the class TestLogSegmentMetadataStoreUpdater method testUpdateLastDLSN.

@Test(timeout = 60000)
public void testUpdateLastDLSN() throws Exception {
    String ledgerPath = "/testUpdateLastDLSN";
    zkc.get().create(ledgerPath, new byte[0], ZooDefs.Ids.OPEN_ACL_UNSAFE, CreateMode.PERSISTENT);
    // Create 1 completed log segment
    LogSegmentMetadata completedLogSegment = DLMTestUtil.completedLogSegment(ledgerPath, 1L, 0L, 99L, 100, 1L, 99L, 0L);
    completedLogSegment.write(zkc);
    // Create 1 inprogress log segment
    LogSegmentMetadata inprogressLogSegment = DLMTestUtil.inprogressLogSegment(ledgerPath, 2L, 100L, 2L);
    inprogressLogSegment.write(zkc);
    DLSN badLastDLSN = new DLSN(99L, 0L, 0L);
    DLSN goodLastDLSN1 = new DLSN(1L, 100L, 0L);
    DLSN goodLastDLSN2 = new DLSN(2L, 200L, 0L);
    LogRecordWithDLSN badRecord = DLMTestUtil.getLogRecordWithDLSNInstance(badLastDLSN, 100L);
    LogRecordWithDLSN goodRecord1 = DLMTestUtil.getLogRecordWithDLSNInstance(goodLastDLSN1, 100L);
    LogRecordWithDLSN goodRecord2 = DLMTestUtil.getLogRecordWithDLSNInstance(goodLastDLSN2, 200L);
    // Dryrun
    MetadataUpdater dryrunUpdater = new DryrunLogSegmentMetadataStoreUpdater(conf, metadataStore);
    try {
        FutureUtils.result(dryrunUpdater.updateLastRecord(completedLogSegment, badRecord));
        fail("Should fail on updating dlsn that in different log segment");
    } catch (IllegalArgumentException iae) {
    // expected
    }
    try {
        FutureUtils.result(dryrunUpdater.updateLastRecord(inprogressLogSegment, goodRecord2));
        fail("Should fail on updating dlsn for an inprogress log segment");
    } catch (IllegalStateException ise) {
    // expected
    }
    LogSegmentMetadata updatedCompletedLogSegment = FutureUtils.result(dryrunUpdater.updateLastRecord(completedLogSegment, goodRecord1));
    assertEquals(goodLastDLSN1, updatedCompletedLogSegment.getLastDLSN());
    assertEquals(goodRecord1.getTransactionId(), updatedCompletedLogSegment.getLastTxId());
    assertTrue(updatedCompletedLogSegment.isRecordLastPositioninThisSegment(goodRecord1));
    Map<Long, LogSegmentMetadata> segmentList = readLogSegments(ledgerPath);
    assertEquals(2, segmentList.size());
    LogSegmentMetadata readCompletedLogSegment = segmentList.get(1L);
    assertNotNull(readCompletedLogSegment);
    assertEquals(completedLogSegment, readCompletedLogSegment);
    LogSegmentMetadata readInprogressLogSegment = segmentList.get(2L);
    assertNotNull(readInprogressLogSegment);
    assertEquals(inprogressLogSegment, readInprogressLogSegment);
    // Fix the last dlsn
    MetadataUpdater updater = LogSegmentMetadataStoreUpdater.createMetadataUpdater(conf, metadataStore);
    try {
        FutureUtils.result(updater.updateLastRecord(completedLogSegment, badRecord));
        fail("Should fail on updating dlsn that in different log segment");
    } catch (IllegalArgumentException iae) {
    // expected
    }
    try {
        FutureUtils.result(updater.updateLastRecord(inprogressLogSegment, goodRecord2));
        fail("Should fail on updating dlsn for an inprogress log segment");
    } catch (IllegalStateException ise) {
    // expected
    }
    updatedCompletedLogSegment = FutureUtils.result(updater.updateLastRecord(completedLogSegment, goodRecord1));
    assertEquals(goodLastDLSN1, updatedCompletedLogSegment.getLastDLSN());
    assertEquals(goodRecord1.getTransactionId(), updatedCompletedLogSegment.getLastTxId());
    assertTrue(updatedCompletedLogSegment.isRecordLastPositioninThisSegment(goodRecord1));
    segmentList = readLogSegments(ledgerPath);
    assertEquals(2, segmentList.size());
    readCompletedLogSegment = segmentList.get(1L);
    assertNotNull(readCompletedLogSegment);
    assertEquals(goodLastDLSN1, readCompletedLogSegment.getLastDLSN());
    assertEquals(goodRecord1.getTransactionId(), readCompletedLogSegment.getLastTxId());
    assertTrue(readCompletedLogSegment.isRecordLastPositioninThisSegment(goodRecord1));
    assertEquals(updatedCompletedLogSegment, readCompletedLogSegment);
    assertEquals(completedLogSegment.getCompletionTime(), readCompletedLogSegment.getCompletionTime());
    assertEquals(completedLogSegment.getFirstTxId(), readCompletedLogSegment.getFirstTxId());
    assertEquals(completedLogSegment.getLedgerId(), readCompletedLogSegment.getLedgerId());
    assertEquals(completedLogSegment.getLogSegmentSequenceNumber(), readCompletedLogSegment.getLogSegmentSequenceNumber());
    assertEquals(completedLogSegment.getRegionId(), readCompletedLogSegment.getRegionId());
    assertEquals(completedLogSegment.getZkPath(), readCompletedLogSegment.getZkPath());
    assertEquals(completedLogSegment.getZNodeName(), readCompletedLogSegment.getZNodeName());
    readInprogressLogSegment = segmentList.get(2L);
    assertNotNull(readInprogressLogSegment);
    assertEquals(inprogressLogSegment, readInprogressLogSegment);
}