Examples with StreamSegmentInformation io.pravega.segmentstore.contracts.StreamSegmentInformation used on opensource projects

Example 1 with StreamSegmentInformation

use of io.pravega.segmentstore.contracts.StreamSegmentInformation in project pravega by pravega.

the class ExtendedS3Storage method doOpenWrite.

private SegmentHandle doOpenWrite(String streamSegmentName) {
    long traceId = LoggerHelpers.traceEnter(log, "openWrite", streamSegmentName);
    StreamSegmentInformation info = doGetStreamSegmentInfo(streamSegmentName);
    ExtendedS3SegmentHandle retHandle;
    if (info.isSealed()) {
        retHandle = ExtendedS3SegmentHandle.getReadHandle(streamSegmentName);
    } else {
        retHandle = ExtendedS3SegmentHandle.getWriteHandle(streamSegmentName);
    }
    LoggerHelpers.traceLeave(log, "openWrite", traceId);
    return retHandle;
}

Example 2 with StreamSegmentInformation

use of io.pravega.segmentstore.contracts.StreamSegmentInformation in project pravega by pravega.

the class ExtendedS3Storage method doOpenWrite.

private SegmentHandle doOpenWrite(String streamSegmentName) {
    long traceId = LoggerHelpers.traceEnter(log, "openWrite", streamSegmentName);
    StreamSegmentInformation info = doGetStreamSegmentInfo(streamSegmentName);
    ExtendedS3SegmentHandle retHandle;
    if (info.isSealed()) {
        retHandle = ExtendedS3SegmentHandle.getReadHandle(streamSegmentName);
    } else {
        retHandle = ExtendedS3SegmentHandle.getWriteHandle(streamSegmentName);
    }
    LoggerHelpers.traceLeave(log, "openWrite", traceId);
    return retHandle;
}

Example 3 with StreamSegmentInformation

use of io.pravega.segmentstore.contracts.StreamSegmentInformation in project pravega by pravega.

the class ExtendedS3Storage method doGetStreamSegmentInfo.

private StreamSegmentInformation doGetStreamSegmentInfo(String streamSegmentName) {
    long traceId = LoggerHelpers.traceEnter(log, "getStreamSegmentInfo", streamSegmentName);
    S3ObjectMetadata result = client.getObjectMetadata(config.getBucket(), config.getPrefix() + streamSegmentName);
    AccessControlList acls = client.getObjectAcl(config.getBucket(), config.getPrefix() + streamSegmentName);
    boolean canWrite = acls.getGrants().stream().anyMatch(grant -> grant.getPermission().compareTo(Permission.WRITE) >= 0);
    StreamSegmentInformation information = StreamSegmentInformation.builder().name(streamSegmentName).length(result.getContentLength()).sealed(!canWrite).lastModified(new ImmutableDate(result.getLastModified().toInstant().toEpochMilli())).build();
    LoggerHelpers.traceLeave(log, "getStreamSegmentInfo", traceId, streamSegmentName);
    return information;
}

Example 4 with StreamSegmentInformation

use of io.pravega.segmentstore.contracts.StreamSegmentInformation in project pravega by pravega.

the class StreamSegmentStoreTestBase method checkReads.

private void checkReads(HashMap<String, ByteArrayOutputStream> segmentContents, StreamSegmentStore store) {
    for (Map.Entry<String, ByteArrayOutputStream> e : segmentContents.entrySet()) {
        String segmentName = e.getKey();
        byte[] expectedData = e.getValue().toByteArray();
        AtomicReference<StreamSegmentInformation> info = new AtomicReference<>((StreamSegmentInformation) store.getStreamSegmentInfo(segmentName, TIMEOUT).join());
        Assert.assertEquals("Unexpected Read Index length for segment " + segmentName, expectedData.length, info.get().getLength());
        AtomicLong expectedCurrentOffset = new AtomicLong(0);
        // We retry a number of times on StreamSegmentNotExists. It is possible that waitForSegmentsInStorage may have
        // returned successfully because it detected the Segment was complete there, but the internal callback to the
        // ReadIndex (completeMerge) may not yet have been executed. The ReadIndex has a mechanism to cope with this,
        // but it only retries once, after a fixed time interval, which is more than generous on any system.
        // However, on very slow systems, it is possible that that callback may take a significant amount of time to even
        // begin executing, hence the trying to read data that was merged from a Transaction may result in a spurious
        // StreamSegmentNotExistsException.
        // This is gracefully handled by retries in AppendProcessor and/or Client, but in this case, we simply have to
        // do the retries ourselves, hoping that the callback eventually executes.
        Retry.withExpBackoff(100, 2, 10, TIMEOUT.toMillis() / 5).retryWhen(ex -> Exceptions.unwrap(ex) instanceof StreamSegmentNotExistsException || info.get().getLength() != info.get().getStorageLength()).run(() -> {
            val latestInfo = (StreamSegmentInformation) store.getStreamSegmentInfo(segmentName, TIMEOUT).join();
            try {
                checkSegmentReads(segmentName, expectedCurrentOffset, info.get().getLength(), store, expectedData);
            } catch (Exception ex2) {
                log.debug("Exception during checkReads", ex2);
            }
            info.set(latestInfo);
            return null;
        });
    }
}

Example 5 with StreamSegmentInformation

use of io.pravega.segmentstore.contracts.StreamSegmentInformation in project pravega by pravega.

the class ExtendedS3Storage method doGetStreamSegmentInfo.

private StreamSegmentInformation doGetStreamSegmentInfo(String streamSegmentName) {
    long traceId = LoggerHelpers.traceEnter(log, "getStreamSegmentInfo", streamSegmentName);
    S3ObjectMetadata result = client.getObjectMetadata(config.getBucket(), config.getRoot() + streamSegmentName);
    AccessControlList acls = client.getObjectAcl(config.getBucket(), config.getRoot() + streamSegmentName);
    boolean canWrite = acls.getGrants().stream().anyMatch(grant -> grant.getPermission().compareTo(Permission.WRITE) >= 0);
    StreamSegmentInformation information = StreamSegmentInformation.builder().name(streamSegmentName).length(result.getContentLength()).sealed(!canWrite).lastModified(new ImmutableDate(result.getLastModified().toInstant().toEpochMilli())).build();
    LoggerHelpers.traceLeave(log, "getStreamSegmentInfo", traceId, streamSegmentName);
    return information;
}