Examples with LogSegmentMetadata com.twitter.distributedlog.LogSegmentMetadata used on opensource projects

Example 1 with LogSegmentMetadata

use of com.twitter.distributedlog.LogSegmentMetadata in project distributedlog by twitter.

the class LogSegmentCache method getLogSegments.

/**
     * Retrieve log segments from the cache.
     *
     * - first sort the log segments in ascending order
     * - do validation and assign corresponding sequence id
     * - apply comparator after validation
     *
     * @param comparator
     *          comparator to sort the returned log segments.
     * @return list of sorted and filtered log segments.
     * @throws UnexpectedException if unexpected condition detected (e.g. ledger sequence number gap)
     */
public List<LogSegmentMetadata> getLogSegments(Comparator<LogSegmentMetadata> comparator) throws UnexpectedException {
    List<LogSegmentMetadata> segmentsToReturn;
    synchronized (logSegments) {
        segmentsToReturn = new ArrayList<LogSegmentMetadata>(logSegments.size());
        segmentsToReturn.addAll(logSegments.values());
    }
    Collections.sort(segmentsToReturn, LogSegmentMetadata.COMPARATOR);
    long startSequenceId = DistributedLogConstants.UNASSIGNED_SEQUENCE_ID;
    LogSegmentMetadata prevSegment = null;
    for (int i = 0; i < segmentsToReturn.size(); i++) {
        LogSegmentMetadata segment = segmentsToReturn.get(i);
        //   states (inprogress vs completed). it could happen during completing log segment without transaction
        if (null != prevSegment && prevSegment.getVersion() >= LogSegmentMetadata.LogSegmentMetadataVersion.VERSION_V2_LEDGER_SEQNO.value && segment.getVersion() >= LogSegmentMetadata.LogSegmentMetadataVersion.VERSION_V2_LEDGER_SEQNO.value && prevSegment.getLogSegmentSequenceNumber() != segment.getLogSegmentSequenceNumber() && prevSegment.getLogSegmentSequenceNumber() + 1 != segment.getLogSegmentSequenceNumber()) {
            LOG.error("{} found ledger sequence number gap between log segment {} and {}", new Object[] { streamName, prevSegment, segment });
            throw new UnexpectedException(streamName + " found ledger sequence number gap between log segment " + prevSegment.getLogSegmentSequenceNumber() + " and " + segment.getLogSegmentSequenceNumber());
        }
        // assign sequence id
        if (!segment.isInProgress()) {
            if (segment.supportsSequenceId()) {
                startSequenceId = segment.getStartSequenceId() + segment.getRecordCount();
                if (null != prevSegment && prevSegment.supportsSequenceId() && prevSegment.getStartSequenceId() > segment.getStartSequenceId()) {
                    LOG.warn("{} found decreasing start sequence id in log segment {}, previous is {}", new Object[] { streamName, segment, prevSegment });
                }
            } else {
                startSequenceId = DistributedLogConstants.UNASSIGNED_SEQUENCE_ID;
            }
        } else {
            if (segment.supportsSequenceId()) {
                LogSegmentMetadata newSegment = segment.mutator().setStartSequenceId(startSequenceId == DistributedLogConstants.UNASSIGNED_SEQUENCE_ID ? 0L : startSequenceId).build();
                segmentsToReturn.set(i, newSegment);
            }
            break;
        }
        prevSegment = segment;
    }
    if (comparator != LogSegmentMetadata.COMPARATOR) {
        Collections.sort(segmentsToReturn, comparator);
    }
    return segmentsToReturn;
}

Example 2 with LogSegmentMetadata

use of com.twitter.distributedlog.LogSegmentMetadata in project distributedlog by twitter.

the class LogSegmentMetadataStoreUpdater method setLogSegmentTruncated.

@Override
public LogSegmentMetadata setLogSegmentTruncated(Transaction<Object> txn, LogSegmentMetadata segment) {
    final LogSegmentMetadata newSegment = segment.mutator().setTruncationStatus(LogSegmentMetadata.TruncationStatus.TRUNCATED).build();
    addNewSegmentAndDeleteOldSegment(txn, newSegment, segment);
    return newSegment;
}

Example 3 with LogSegmentMetadata

use of com.twitter.distributedlog.LogSegmentMetadata in project distributedlog by twitter.

the class LogSegmentMetadataStoreUpdater method updateLastRecord.

@Override
public Future<LogSegmentMetadata> updateLastRecord(LogSegmentMetadata segment, LogRecordWithDLSN record) {
    DLSN dlsn = record.getDlsn();
    Preconditions.checkState(!segment.isInProgress(), "Updating last dlsn for an inprogress log segment isn't supported.");
    Preconditions.checkArgument(segment.isDLSNinThisSegment(dlsn), "DLSN " + dlsn + " doesn't belong to segment " + segment);
    final LogSegmentMetadata newSegment = segment.mutator().setLastDLSN(dlsn).setLastTxId(record.getTransactionId()).setRecordCount(record).build();
    return updateSegmentMetadata(newSegment);
}

Example 4 with LogSegmentMetadata

use of com.twitter.distributedlog.LogSegmentMetadata in project distributedlog by twitter.

the class LogSegmentMetadataStoreUpdater method setLogSegmentPartiallyTruncated.

@Override
public LogSegmentMetadata setLogSegmentPartiallyTruncated(Transaction<Object> txn, LogSegmentMetadata segment, DLSN minActiveDLSN) {
    final LogSegmentMetadata newSegment = segment.mutator().setTruncationStatus(LogSegmentMetadata.TruncationStatus.PARTIALLY_TRUNCATED).setMinActiveDLSN(minActiveDLSN).build();
    addNewSegmentAndDeleteOldSegment(txn, newSegment, segment);
    return newSegment;
}

Example 5 with LogSegmentMetadata

use of com.twitter.distributedlog.LogSegmentMetadata in project distributedlog by twitter.

the class TestLogSegmentCache method testUpdate.

@Test(timeout = 60000)
public void testUpdate() {
    LogSegmentCache cache = new LogSegmentCache("test-update");
    // add 5 completed log segments
    for (int i = 1; i <= 5; i++) {
        LogSegmentMetadata metadata = DLMTestUtil.completedLogSegment("/segment" + i, i, i, i * 100L, 100, i, 99L, 0L);
        String name = DLMTestUtil.completedLedgerZNodeNameWithLogSegmentSequenceNumber(i);
        cache.add(name, metadata);
    }
    // add one inprogress log segment
    LogSegmentMetadata inprogress = DLMTestUtil.inprogressLogSegment("/inprogress-6", 6, 600L, 6);
    String name = DLMTestUtil.inprogressZNodeName(6);
    cache.add(name, inprogress);
    // deleted first 2 completed log segments and completed the last one
    Set<String> segmentRemoved = Sets.newHashSet();
    for (int i = 1; i <= 2; i++) {
        segmentRemoved.add(DLMTestUtil.completedLedgerZNodeNameWithLogSegmentSequenceNumber(i));
    }
    segmentRemoved.add((DLMTestUtil.inprogressZNodeName(6)));
    Set<String> segmentReceived = Sets.newHashSet();
    Map<String, LogSegmentMetadata> segmentAdded = Maps.newHashMap();
    for (int i = 3; i <= 6; i++) {
        segmentReceived.add(DLMTestUtil.completedLedgerZNodeNameWithLogSegmentSequenceNumber(i));
        if (i == 6) {
            segmentAdded.put(DLMTestUtil.completedLedgerZNodeNameWithLogSegmentSequenceNumber(i), DLMTestUtil.completedLogSegment("/segment" + i, i, i, i * 100L, 100, i, 99L, 0L));
        }
    }
    // update the cache
    cache.update(segmentRemoved, segmentAdded);
    for (String segment : segmentRemoved) {
        assertNull("Segment " + segment + " should be removed.", cache.get(segment));
    }
    for (String segment : segmentReceived) {
        assertNotNull("Segment " + segment + " should not be removed", cache.get(segment));
    }
    for (Map.Entry<String, LogSegmentMetadata> entry : segmentAdded.entrySet()) {
        assertEquals("Segment " + entry.getKey() + " should be added.", entry.getValue(), entry.getValue());
    }
}