Examples with LogSegmentMetadata org.apache.distributedlog.LogSegmentMetadata used on opensource projects

Example 46 with LogSegmentMetadata

use of org.apache.distributedlog.LogSegmentMetadata in project bookkeeper by apache.

the class BKLogSegmentEntryReader method openComplete.

@Override
public void openComplete(int rc, LedgerHandle lh, Object ctx) {
    LogSegmentMetadata segment = (LogSegmentMetadata) ctx;
    if (BKException.Code.OK != rc) {
        // fail current reader or retry opening the reader
        failOrRetryOpenLedger(rc, segment);
        return;
    }
    // switch to new ledger handle if the log segment is moved to completed.
    CacheEntry longPollRead = null;
    synchronized (this) {
        if (isClosed()) {
            lh.asyncClose(new AsyncCallback.CloseCallback() {

                @Override
                public void closeComplete(int rc, LedgerHandle lh, Object ctx) {
                    logger.debug("Close the open ledger {} since the log segment reader is already closed", lh.getId());
                }
            }, null);
            return;
        }
        this.metadata = segment;
        this.lh = lh;
        this.openLedgerHandles.add(lh);
        longPollRead = outstandingLongPoll;
    }
    if (null != longPollRead) {
        // reissue the long poll read when the log segment state is changed
        issueRead(longPollRead);
    }
    // notify readers
    notifyReaders();
}

Example 47 with LogSegmentMetadata

use of org.apache.distributedlog.LogSegmentMetadata in project bookkeeper by apache.

the class DLAuditor method calculateStreamSpaceUsage.

private long calculateStreamSpaceUsage(final Namespace namespace, final String stream) throws IOException {
    DistributedLogManager dlm = namespace.openLog(stream);
    long totalBytes = 0;
    try {
        List<LogSegmentMetadata> segments = dlm.getLogSegments();
        for (LogSegmentMetadata segment : segments) {
            try {
                LedgerHandle lh = getBookKeeperClient(namespace).get().openLedgerNoRecovery(segment.getLogSegmentId(), BookKeeper.DigestType.CRC32, conf.getBKDigestPW().getBytes(UTF_8));
                totalBytes += lh.getLength();
                lh.close();
            } catch (BKException e) {
                logger.error("Failed to open ledger {} : ", segment.getLogSegmentId(), e);
                throw new IOException("Failed to open ledger " + segment.getLogSegmentId(), e);
            } catch (InterruptedException e) {
                logger.warn("Interrupted on opening ledger {} : ", segment.getLogSegmentId(), e);
                Thread.currentThread().interrupt();
                throw new DLInterruptedException("Interrupted on opening ledger " + segment.getLogSegmentId(), e);
            }
        }
    } finally {
        dlm.close();
    }
    return totalBytes;
}

Example 48 with LogSegmentMetadata

use of org.apache.distributedlog.LogSegmentMetadata in project bookkeeper by apache.

the class DLAuditor method collectLedgersFromStream.

private List<Long> collectLedgersFromStream(Namespace namespace, String stream, Set<Long> ledgers) throws IOException {
    DistributedLogManager dlm = namespace.openLog(stream);
    try {
        List<LogSegmentMetadata> segments = dlm.getLogSegments();
        List<Long> sLedgers = new ArrayList<Long>();
        for (LogSegmentMetadata segment : segments) {
            synchronized (ledgers) {
                ledgers.add(segment.getLogSegmentId());
            }
            sLedgers.add(segment.getLogSegmentId());
        }
        return sLedgers;
    } finally {
        dlm.close();
    }
}

Example 49 with LogSegmentMetadata

use of org.apache.distributedlog.LogSegmentMetadata in project bookkeeper by apache.

the class PerStreamLogSegmentCache method getLogSegments.

/**
 * Retrieve log segments from the cache.
 *- first sort the log segments in ascending order
 * - do validation and assign corresponding sequence id
 * - apply comparator after validation
 *
 * @param comparator
 *          comparator to sort the returned log segments.
 * @return list of sorted and filtered log segments.
 * @throws UnexpectedException if unexpected condition detected (e.g. ledger sequence number gap)
 */
public List<LogSegmentMetadata> getLogSegments(Comparator<LogSegmentMetadata> comparator) throws UnexpectedException {
    List<LogSegmentMetadata> segmentsToReturn;
    synchronized (logSegments) {
        segmentsToReturn = new ArrayList<LogSegmentMetadata>(logSegments.size());
        segmentsToReturn.addAll(logSegments.values());
    }
    Collections.sort(segmentsToReturn, LogSegmentMetadata.COMPARATOR);
    LogSegmentMetadata prevSegment = null;
    if (validateLogSegmentSequenceNumber) {
        // validation ledger sequence number to ensure the log segments are unique.
        for (int i = 0; i < segmentsToReturn.size(); i++) {
            LogSegmentMetadata segment = segmentsToReturn.get(i);
            if (null != prevSegment && prevSegment.getVersion() >= LogSegmentMetadata.LogSegmentMetadataVersion.VERSION_V2_LEDGER_SEQNO.value && segment.getVersion() >= LogSegmentMetadata.LogSegmentMetadataVersion.VERSION_V2_LEDGER_SEQNO.value && prevSegment.getLogSegmentSequenceNumber() + 1 != segment.getLogSegmentSequenceNumber()) {
                LOG.error("{} found ledger sequence number gap between log segment {} and {}", new Object[] { streamName, prevSegment, segment });
                throw new UnexpectedException(streamName + " found ledger sequence number gap between log segment " + prevSegment.getLogSegmentSequenceNumber() + " and " + segment.getLogSegmentSequenceNumber());
            }
            prevSegment = segment;
        }
    }
    prevSegment = null;
    long startSequenceId = DistributedLogConstants.UNASSIGNED_SEQUENCE_ID;
    for (int i = 0; i < segmentsToReturn.size(); i++) {
        LogSegmentMetadata segment = segmentsToReturn.get(i);
        // assign sequence id
        if (!segment.isInProgress()) {
            if (segment.supportsSequenceId()) {
                startSequenceId = segment.getStartSequenceId() + segment.getRecordCount();
                if (null != prevSegment && prevSegment.supportsSequenceId() && prevSegment.getStartSequenceId() > segment.getStartSequenceId()) {
                    LOG.warn("{} found decreasing start sequence id in log segment {}, previous is {}", new Object[] { streamName, segment, prevSegment });
                }
            } else {
                startSequenceId = DistributedLogConstants.UNASSIGNED_SEQUENCE_ID;
            }
        } else {
            if (segment.supportsSequenceId()) {
                LogSegmentMetadata newSegment = segment.mutator().setStartSequenceId(startSequenceId == DistributedLogConstants.UNASSIGNED_SEQUENCE_ID ? 0L : startSequenceId).build();
                segmentsToReturn.set(i, newSegment);
            }
            break;
        }
        prevSegment = segment;
    }
    if (comparator != LogSegmentMetadata.COMPARATOR) {
        Collections.sort(segmentsToReturn, comparator);
    }
    return segmentsToReturn;
}

Example 50 with LogSegmentMetadata

use of org.apache.distributedlog.LogSegmentMetadata in project bookkeeper by apache.

the class LogSegmentMetadataStoreUpdater method changeSequenceNumber.

@Override
public CompletableFuture<LogSegmentMetadata> changeSequenceNumber(LogSegmentMetadata segment, long logSegmentSeqNo) {
    String newZkPath = segment.getZkPath().replace(formatLogSegmentSequenceNumber(segment.getLogSegmentSequenceNumber()), formatLogSegmentSequenceNumber(logSegmentSeqNo));
    final LogSegmentMetadata newSegment = segment.mutator().setLogSegmentSequenceNumber(logSegmentSeqNo).setZkPath(newZkPath).build();
    return addNewSegmentAndDeleteOldSegment(newSegment, segment);
}