Examples with LogSegmentMetadata com.twitter.distributedlog.LogSegmentMetadata used on opensource projects

Example 36 with LogSegmentMetadata

use of com.twitter.distributedlog.LogSegmentMetadata in project distributedlog by twitter.

the class TestLogSegmentCache method testDiff.

@Test(timeout = 60000)
public void testDiff() {
    LogSegmentCache cache = new LogSegmentCache("test-diff");
    // add 5 completed log segments
    for (int i = 1; i <= 5; i++) {
        LogSegmentMetadata metadata = DLMTestUtil.completedLogSegment("/segment" + i, i, i, i * 100L, 100, i, 99L, 0L);
        String name = DLMTestUtil.completedLedgerZNodeNameWithLogSegmentSequenceNumber(i);
        cache.add(name, metadata);
    }
    // add one inprogress log segment
    LogSegmentMetadata inprogress = DLMTestUtil.inprogressLogSegment("/inprogress-6", 6, 600L, 6);
    String name = DLMTestUtil.inprogressZNodeName(6);
    cache.add(name, inprogress);
    // deleted first 2 completed log segments and completed the last one
    Set<String> segmentRemoved = Sets.newHashSet();
    for (int i = 1; i <= 2; i++) {
        segmentRemoved.add(DLMTestUtil.completedLedgerZNodeNameWithLogSegmentSequenceNumber(i));
    }
    segmentRemoved.add((DLMTestUtil.inprogressZNodeName(6)));
    Set<String> segmentReceived = Sets.newHashSet();
    Set<String> segmentAdded = Sets.newHashSet();
    for (int i = 3; i <= 6; i++) {
        segmentReceived.add(DLMTestUtil.completedLedgerZNodeNameWithLogSegmentSequenceNumber(i));
        if (i == 6) {
            segmentAdded.add(DLMTestUtil.completedLedgerZNodeNameWithLogSegmentSequenceNumber(i));
        }
    }
    Pair<Set<String>, Set<String>> segmentChanges = cache.diff(segmentReceived);
    assertTrue("Should remove " + segmentRemoved + ", but removed " + segmentChanges.getRight(), Sets.difference(segmentRemoved, segmentChanges.getRight()).isEmpty());
    assertTrue("Should add " + segmentAdded + ", but added " + segmentChanges.getLeft(), Sets.difference(segmentAdded, segmentChanges.getLeft()).isEmpty());
}

Example 37 with LogSegmentMetadata

use of com.twitter.distributedlog.LogSegmentMetadata in project distributedlog by twitter.

the class DLUtils method nextLogSegmentSequenceNumber.

/**
     * Assign next log segment sequence number based on a decreasing list of log segments.
     *
     * @param segmentListDesc
     *          a decreasing list of log segments
     * @return null if no log segments was assigned a sequence number in <code>segmentListDesc</code>.
     *         otherwise, return next log segment sequence number
     */
public static Long nextLogSegmentSequenceNumber(List<LogSegmentMetadata> segmentListDesc) {
    int lastAssignedLogSegmentIdx = -1;
    Long lastAssignedLogSegmentSeqNo = null;
    Long nextLogSegmentSeqNo = null;
    for (int i = 0; i < segmentListDesc.size(); i++) {
        LogSegmentMetadata metadata = segmentListDesc.get(i);
        if (LogSegmentMetadata.supportsLogSegmentSequenceNo(metadata.getVersion())) {
            lastAssignedLogSegmentSeqNo = metadata.getLogSegmentSequenceNumber();
            lastAssignedLogSegmentIdx = i;
            break;
        }
    }
    if (null != lastAssignedLogSegmentSeqNo) {
        // latest log segment is assigned with a sequence number, start with next sequence number
        nextLogSegmentSeqNo = lastAssignedLogSegmentSeqNo + lastAssignedLogSegmentIdx + 1;
    }
    return nextLogSegmentSeqNo;
}

Example 38 with LogSegmentMetadata

use of com.twitter.distributedlog.LogSegmentMetadata in project distributedlog by twitter.

the class DLUtils method findLogSegmentNotLessThanTxnId.

/**
     * Find the log segment whose transaction ids are not less than provided <code>transactionId</code>.
     *
     * @param segments
     *          segments to search
     * @param transactionId
     *          transaction id to find
     * @return the first log segment whose transaction ids are not less than <code>transactionId</code>.
     */
public static int findLogSegmentNotLessThanTxnId(List<LogSegmentMetadata> segments, long transactionId) {
    int found = -1;
    for (int i = segments.size() - 1; i >= 0; i--) {
        LogSegmentMetadata segment = segments.get(i);
        if (segment.getFirstTxId() <= transactionId) {
            found = i;
            break;
        }
    }
    if (found <= -1) {
        return -1;
    }
    if (found == 0 && segments.get(0).getFirstTxId() == transactionId) {
        return 0;
    }
    LogSegmentMetadata foundSegment = segments.get(found);
    if (foundSegment.getFirstTxId() == transactionId) {
        for (int i = found - 1; i >= 0; i--) {
            LogSegmentMetadata segment = segments.get(i);
            if (segment.isInProgress()) {
                break;
            }
            if (segment.getLastTxId() < transactionId) {
                break;
            }
            found = i;
        }
        return found;
    } else {
        if (foundSegment.isInProgress() || found == segments.size() - 1) {
            return found;
        }
        if (foundSegment.getLastTxId() >= transactionId) {
            return found;
        }
        return found + 1;
    }
}

Example 39 with LogSegmentMetadata

use of com.twitter.distributedlog.LogSegmentMetadata in project distributedlog by twitter.

the class LogSegmentMetadataStoreUpdater method changeSequenceNumber.

@Override
public Future<LogSegmentMetadata> changeSequenceNumber(LogSegmentMetadata segment, long logSegmentSeqNo) {
    String newZkPath = segment.getZkPath().replace(formatLogSegmentSequenceNumber(segment.getLogSegmentSequenceNumber()), formatLogSegmentSequenceNumber(logSegmentSeqNo));
    final LogSegmentMetadata newSegment = segment.mutator().setLogSegmentSequenceNumber(logSegmentSeqNo).setZkPath(newZkPath).build();
    return addNewSegmentAndDeleteOldSegment(newSegment, segment);
}

Example 40 with LogSegmentMetadata

use of com.twitter.distributedlog.LogSegmentMetadata in project distributedlog by twitter.

the class DistributedLogAdmin method fixInprogressSegmentWithLowerSequenceNumber.

/**
     * Fix inprogress segment with lower ledger sequence number.
     *
     * @param factory
     *          dlm factory.
     * @param metadataUpdater
     *          metadata updater.
     * @param streamName
     *          stream name.
     * @param verbose
     *          print verbose messages.
     * @param interactive
     *          is confirmation needed before executing actual action.
     * @throws IOException
     */
public static void fixInprogressSegmentWithLowerSequenceNumber(final com.twitter.distributedlog.DistributedLogManagerFactory factory, final MetadataUpdater metadataUpdater, final String streamName, final boolean verbose, final boolean interactive) throws IOException {
    DistributedLogManager dlm = factory.createDistributedLogManagerWithSharedClients(streamName);
    try {
        List<LogSegmentMetadata> segments = dlm.getLogSegments();
        if (verbose) {
            System.out.println("LogSegments for " + streamName + " : ");
            for (LogSegmentMetadata segment : segments) {
                System.out.println(segment.getLogSegmentSequenceNumber() + "\t: " + segment);
            }
        }
        LOG.info("Get log segments for {} : {}", streamName, segments);
        // validate log segments
        long maxCompletedLogSegmentSequenceNumber = -1L;
        LogSegmentMetadata inprogressSegment = null;
        for (LogSegmentMetadata segment : segments) {
            if (!segment.isInProgress()) {
                maxCompletedLogSegmentSequenceNumber = Math.max(maxCompletedLogSegmentSequenceNumber, segment.getLogSegmentSequenceNumber());
            } else {
                // we already found an inprogress segment
                if (null != inprogressSegment) {
                    throw new DLIllegalStateException("Multiple inprogress segments found for stream " + streamName + " : " + segments);
                }
                inprogressSegment = segment;
            }
        }
        if (null == inprogressSegment || inprogressSegment.getLogSegmentSequenceNumber() > maxCompletedLogSegmentSequenceNumber) {
            // nothing to fix
            return;
        }
        final long newLogSegmentSequenceNumber = maxCompletedLogSegmentSequenceNumber + 1;
        if (interactive && !IOUtils.confirmPrompt("Confirm to fix (Y/N), Ctrl+C to break : ")) {
            return;
        }
        final LogSegmentMetadata newSegment = FutureUtils.result(metadataUpdater.changeSequenceNumber(inprogressSegment, newLogSegmentSequenceNumber));
        LOG.info("Fixed {} : {} -> {} ", new Object[] { streamName, inprogressSegment, newSegment });
        if (verbose) {
            System.out.println("Fixed " + streamName + " : " + inprogressSegment.getZNodeName() + " -> " + newSegment.getZNodeName());
            System.out.println("\t old: " + inprogressSegment);
            System.out.println("\t new: " + newSegment);
            System.out.println();
        }
    } finally {
        dlm.close();
    }
}