Examples with LogSegmentMetadata org.apache.distributedlog.LogSegmentMetadata used on opensource projects

Example 1 with LogSegmentMetadata

use of org.apache.distributedlog.LogSegmentMetadata in project bookkeeper by apache.

the class PerStreamLogSegmentCache method remove.

/**
 * Remove log segment <code>name</code> from the cache.
 *
 * @param name
 *          name of the log segment.
 * @return log segment metadata.
 */
public LogSegmentMetadata remove(String name) {
    synchronized (logSegments) {
        LogSegmentMetadata metadata = logSegments.remove(name);
        if (null != metadata) {
            lid2LogSegments.remove(metadata.getLogSegmentId(), metadata);
            LOG.debug("Removed log segment ({} : {}) from cache.", name, metadata);
        }
        return metadata;
    }
}

Example 2 with LogSegmentMetadata

use of org.apache.distributedlog.LogSegmentMetadata in project bookkeeper by apache.

the class PerStreamLogSegmentCache method add.

/**
 * Add the segment <i>metadata</i> for <i>name</i> in the cache.
 *
 * @param name
 *          segment name.
 * @param metadata
 *          segment metadata.
 */
public void add(String name, LogSegmentMetadata metadata) {
    synchronized (logSegments) {
        if (!logSegments.containsKey(name)) {
            logSegments.put(name, metadata);
            LOG.info("{} added log segment ({} : {}) to cache.", new Object[] { streamName, name, metadata });
        }
        LogSegmentMetadata oldMetadata = lid2LogSegments.remove(metadata.getLogSegmentId());
        if (null == oldMetadata) {
            lid2LogSegments.put(metadata.getLogSegmentId(), metadata);
        } else {
            if (oldMetadata.isInProgress() && !metadata.isInProgress()) {
                lid2LogSegments.put(metadata.getLogSegmentId(), metadata);
            } else {
                lid2LogSegments.put(oldMetadata.getLogSegmentId(), oldMetadata);
            }
        }
    }
}

Example 3 with LogSegmentMetadata

use of org.apache.distributedlog.LogSegmentMetadata in project bookkeeper by apache.

the class DistributedLogAdmin method checkStream.

private static StreamCandidate checkStream(final Namespace namespace, final String streamName, final OrderedScheduler scheduler) throws IOException {
    DistributedLogManager dlm = namespace.openLog(streamName);
    try {
        List<LogSegmentMetadata> segments = dlm.getLogSegments();
        if (segments.isEmpty()) {
            return null;
        }
        List<CompletableFuture<LogSegmentCandidate>> futures = new ArrayList<CompletableFuture<LogSegmentCandidate>>(segments.size());
        for (LogSegmentMetadata segment : segments) {
            futures.add(checkLogSegment(namespace, streamName, segment, scheduler));
        }
        List<LogSegmentCandidate> segmentCandidates;
        try {
            segmentCandidates = FutureUtils.result(FutureUtils.collect(futures));
        } catch (Exception e) {
            throw new IOException("Failed on checking stream " + streamName, e);
        }
        StreamCandidate streamCandidate = new StreamCandidate(streamName);
        for (LogSegmentCandidate segmentCandidate : segmentCandidates) {
            if (null != segmentCandidate) {
                streamCandidate.addLogSegmentCandidate(segmentCandidate);
            }
        }
        if (streamCandidate.segmentCandidates.isEmpty()) {
            return null;
        }
        return streamCandidate;
    } finally {
        dlm.close();
    }
}

Example 4 with LogSegmentMetadata

use of org.apache.distributedlog.LogSegmentMetadata in project bookkeeper by apache.

the class DistributedLogAdmin method fixInprogressSegmentWithLowerSequenceNumber.

/**
 * Fix inprogress segment with lower ledger sequence number.
 *
 * @param namespace
 *          dl namespace
 * @param metadataUpdater
 *          metadata updater.
 * @param streamName
 *          stream name.
 * @param verbose
 *          print verbose messages.
 * @param interactive
 *          is confirmation needed before executing actual action.
 * @throws IOException
 */
public static void fixInprogressSegmentWithLowerSequenceNumber(final Namespace namespace, final MetadataUpdater metadataUpdater, final String streamName, final boolean verbose, final boolean interactive) throws Exception {
    DistributedLogManager dlm = namespace.openLog(streamName);
    try {
        List<LogSegmentMetadata> segments = dlm.getLogSegments();
        if (verbose) {
            System.out.println("LogSegments for " + streamName + " : ");
            for (LogSegmentMetadata segment : segments) {
                System.out.println(segment.getLogSegmentSequenceNumber() + "\t: " + segment);
            }
        }
        LOG.info("Get log segments for {} : {}", streamName, segments);
        // validate log segments
        long maxCompletedLogSegmentSequenceNumber = -1L;
        LogSegmentMetadata inprogressSegment = null;
        for (LogSegmentMetadata segment : segments) {
            if (!segment.isInProgress()) {
                maxCompletedLogSegmentSequenceNumber = Math.max(maxCompletedLogSegmentSequenceNumber, segment.getLogSegmentSequenceNumber());
            } else {
                // we already found an inprogress segment
                if (null != inprogressSegment) {
                    throw new DLIllegalStateException("Multiple inprogress segments found for stream " + streamName + " : " + segments);
                }
                inprogressSegment = segment;
            }
        }
        if (null == inprogressSegment || inprogressSegment.getLogSegmentSequenceNumber() > maxCompletedLogSegmentSequenceNumber) {
            // nothing to fix
            return;
        }
        final long newLogSegmentSequenceNumber = maxCompletedLogSegmentSequenceNumber + 1;
        if (interactive && !IOUtils.confirmPrompt("Confirm to fix (Y/N), Ctrl+C to break : ")) {
            return;
        }
        final LogSegmentMetadata newSegment = FutureUtils.result(metadataUpdater.changeSequenceNumber(inprogressSegment, newLogSegmentSequenceNumber));
        LOG.info("Fixed {} : {} -> {} ", new Object[] { streamName, inprogressSegment, newSegment });
        if (verbose) {
            System.out.println("Fixed " + streamName + " : " + inprogressSegment.getZNodeName() + " -> " + newSegment.getZNodeName());
            System.out.println("\t old: " + inprogressSegment);
            System.out.println("\t new: " + newSegment);
            System.out.println();
        }
    } finally {
        dlm.close();
    }
}

Example 5 with LogSegmentMetadata

use of org.apache.distributedlog.LogSegmentMetadata in project bookkeeper by apache.

the class ZKLogStreamMetadataStore method renameLogMetadata.

private CompletableFuture<Void> renameLogMetadata(URI uri, LogMetadataForWriter oldMetadata, String newStreamName) {
    final LinkedList<Op> createOps = Lists.newLinkedList();
    final LinkedList<Op> deleteOps = Lists.newLinkedList();
    List<ACL> acls = zooKeeperClient.getDefaultACL();
    // get the root path
    String oldRootPath = oldMetadata.getLogRootPath();
    String newRootPath = LogMetadata.getLogRootPath(uri, newStreamName, conf.getUnpartitionedStreamName());
    // 0. the log path
    deleteOps.addFirst(Op.delete(LogMetadata.getLogStreamPath(uri, oldMetadata.getLogName()), -1));
    // 1. the root path
    createOps.addLast(Op.create(newRootPath, EMPTY_BYTES, acls, CreateMode.PERSISTENT));
    deleteOps.addFirst(Op.delete(oldRootPath, -1));
    // 2. max id
    Versioned<byte[]> maxTxIdData = oldMetadata.getMaxTxIdData();
    deleteOldPathAndCreateNewPath(oldRootPath, MAX_TXID_PATH, maxTxIdData, newRootPath, DLUtils.serializeTransactionId(0L), acls, createOps, deleteOps);
    // 3. version
    createOps.addLast(Op.create(newRootPath + VERSION_PATH, intToBytes(LAYOUT_VERSION), acls, CreateMode.PERSISTENT));
    deleteOps.addFirst(Op.delete(oldRootPath + VERSION_PATH, -1));
    // 4. lock path (NOTE: if the stream is locked by a writer, then the delete will fail as you can not
    // delete the lock path if children is not empty.
    createOps.addLast(Op.create(newRootPath + LOCK_PATH, EMPTY_BYTES, acls, CreateMode.PERSISTENT));
    deleteOps.addFirst(Op.delete(oldRootPath + LOCK_PATH, -1));
    // 5. read lock path (NOTE: same reason as the write lock)
    createOps.addLast(Op.create(newRootPath + READ_LOCK_PATH, EMPTY_BYTES, acls, CreateMode.PERSISTENT));
    deleteOps.addFirst(Op.delete(oldRootPath + READ_LOCK_PATH, -1));
    // 6. allocation path
    Versioned<byte[]> allocationData = oldMetadata.getAllocationData();
    deleteOldPathAndCreateNewPath(oldRootPath, ALLOCATION_PATH, allocationData, newRootPath, EMPTY_BYTES, acls, createOps, deleteOps);
    // 7. log segments
    Versioned<byte[]> maxLSSNData = oldMetadata.getMaxLSSNData();
    deleteOldPathAndCreateNewPath(oldRootPath, LOGSEGMENTS_PATH, maxLSSNData, newRootPath, DLUtils.serializeLogSegmentSequenceNumber(UNASSIGNED_LOGSEGMENT_SEQNO), acls, createOps, deleteOps);
    // 8. copy the log segments
    CompletableFuture<List<LogSegmentMetadata>> segmentsFuture;
    if (pathExists(maxLSSNData)) {
        segmentsFuture = getLogSegments(zooKeeperClient, oldRootPath + LOGSEGMENTS_PATH);
    } else {
        segmentsFuture = FutureUtils.value(Collections.emptyList());
    }
    return segmentsFuture.thenApply(segments -> {
        for (LogSegmentMetadata segment : segments) {
            deleteOldSegmentAndCreateNewSegment(segment, newRootPath + LOGSEGMENTS_PATH, acls, createOps, deleteOps);
        }
        return null;
    }).thenCompose(ignored -> getMissingPaths(zooKeeperClient, uri, newStreamName)).thenCompose(paths -> {
        for (String path : paths) {
            createOps.addFirst(Op.create(path, EMPTY_BYTES, acls, CreateMode.PERSISTENT));
        }
        return executeRenameTxn(oldRootPath, newRootPath, createOps, deleteOps);
    });
}