Search in sources :

Example 6 with Segment

use of io.pravega.client.segment.impl.Segment in project pravega by pravega.

the class ControllerImpl method getSegmentsAtTime.

@Override
public CompletableFuture<Map<Segment, Long>> getSegmentsAtTime(final Stream stream, final long timestamp) {
    Exceptions.checkNotClosed(closed.get(), this);
    Preconditions.checkNotNull(stream, "stream");
    long traceId = LoggerHelpers.traceEnter(log, "getSegmentsAtTime", stream, timestamp);
    final CompletableFuture<SegmentsAtTime> result = this.retryConfig.runAsync(() -> {
        RPCAsyncCallback<SegmentsAtTime> callback = new RPCAsyncCallback<>();
        StreamInfo streamInfo = ModelHelper.createStreamInfo(stream.getScope(), stream.getStreamName());
        GetSegmentsRequest request = GetSegmentsRequest.newBuilder().setStreamInfo(streamInfo).setTimestamp(timestamp).build();
        client.getSegments(request, callback);
        return callback.getFuture();
    }, this.executor);
    return result.thenApply(segments -> {
        log.debug("Received the following data from the controller {}", segments.getSegmentsList());
        return segments.getSegmentsList().stream().collect(Collectors.toMap(location -> ModelHelper.encode(location.getSegmentId()), location -> location.getOffset()));
    }).whenComplete((x, e) -> {
        if (e != null) {
            log.warn("getSegmentsAtTime failed: ", e);
        }
        LoggerHelpers.traceLeave(log, "getSegmentsAtTime", traceId);
    });
}
Also used : StreamCut(io.pravega.client.stream.StreamCut) ManagedChannel(io.grpc.ManagedChannel) Retry(io.pravega.common.util.Retry) NegotiationType(io.grpc.netty.NegotiationType) MoreCallCredentials(io.grpc.auth.MoreCallCredentials) StreamConfiguration(io.pravega.client.stream.StreamConfiguration) PingTxnRequest(io.pravega.controller.stream.api.grpc.v1.Controller.PingTxnRequest) SegmentRanges(io.pravega.controller.stream.api.grpc.v1.Controller.SegmentRanges) GetSegmentsRequest(io.pravega.controller.stream.api.grpc.v1.Controller.GetSegmentsRequest) StreamObserver(io.grpc.stub.StreamObserver) Stream(io.pravega.client.stream.Stream) Map(java.util.Map) Collectors.summarizingInt(java.util.stream.Collectors.summarizingInt) PravegaNodeUri(io.pravega.shared.protocol.netty.PravegaNodeUri) Transaction(io.pravega.client.stream.Transaction) DeleteScopeStatus(io.pravega.controller.stream.api.grpc.v1.Controller.DeleteScopeStatus) CreateStreamStatus(io.pravega.controller.stream.api.grpc.v1.Controller.CreateStreamStatus) Collection(java.util.Collection) Set(java.util.Set) CompletionException(java.util.concurrent.CompletionException) UUID(java.util.UUID) NavigableMap(java.util.NavigableMap) Collectors(java.util.stream.Collectors) List(java.util.List) SSLException(javax.net.ssl.SSLException) Slf4j(lombok.extern.slf4j.Slf4j) CreateTxnResponse(io.pravega.controller.stream.api.grpc.v1.Controller.CreateTxnResponse) PingTxnStatus(io.pravega.controller.stream.api.grpc.v1.Controller.PingTxnStatus) Futures(io.pravega.common.concurrent.Futures) Segment(io.pravega.client.segment.impl.Segment) CreateScopeStatus(io.pravega.controller.stream.api.grpc.v1.Controller.CreateScopeStatus) ScaleStatusRequest(io.pravega.controller.stream.api.grpc.v1.Controller.ScaleStatusRequest) Exceptions(io.pravega.common.Exceptions) AtomicBoolean(java.util.concurrent.atomic.AtomicBoolean) ScaleStatusResponse(io.pravega.controller.stream.api.grpc.v1.Controller.ScaleStatusResponse) ScopeInfo(io.pravega.controller.stream.api.grpc.v1.Controller.ScopeInfo) TxnState(io.pravega.controller.stream.api.grpc.v1.Controller.TxnState) HashMap(java.util.HashMap) CompletableFuture(java.util.concurrent.CompletableFuture) SuccessorResponse(io.pravega.controller.stream.api.grpc.v1.Controller.SuccessorResponse) PingFailedException(io.pravega.client.stream.PingFailedException) ArrayList(java.util.ArrayList) HashSet(java.util.HashSet) Strings(com.google.common.base.Strings) NodeUri(io.pravega.controller.stream.api.grpc.v1.Controller.NodeUri) ScaleRequest(io.pravega.controller.stream.api.grpc.v1.Controller.ScaleRequest) CreateTxnRequest(io.pravega.controller.stream.api.grpc.v1.Controller.CreateTxnRequest) StreamInfo(io.pravega.controller.stream.api.grpc.v1.Controller.StreamInfo) DeleteStreamStatus(io.pravega.controller.stream.api.grpc.v1.Controller.DeleteStreamStatus) ScheduledExecutorService(java.util.concurrent.ScheduledExecutorService) ControllerServiceGrpc(io.pravega.controller.stream.api.grpc.v1.ControllerServiceGrpc) ScaleResponse(io.pravega.controller.stream.api.grpc.v1.Controller.ScaleResponse) LoggerHelpers(io.pravega.common.LoggerHelpers) Futures.getAndHandleExceptions(io.pravega.common.concurrent.Futures.getAndHandleExceptions) lombok.val(lombok.val) SegmentRange(io.pravega.controller.stream.api.grpc.v1.Controller.SegmentRange) File(java.io.File) StatusRuntimeException(io.grpc.StatusRuntimeException) NettyChannelBuilder(io.grpc.netty.NettyChannelBuilder) TimeUnit(java.util.concurrent.TimeUnit) ManagedChannelBuilder(io.grpc.ManagedChannelBuilder) TreeMap(java.util.TreeMap) UpdateStreamStatus(io.pravega.controller.stream.api.grpc.v1.Controller.UpdateStreamStatus) TxnStatus(io.pravega.controller.stream.api.grpc.v1.Controller.TxnStatus) SslContextBuilder(io.netty.handler.ssl.SslContextBuilder) InvalidStreamException(io.pravega.client.stream.InvalidStreamException) Preconditions(com.google.common.base.Preconditions) RoundRobinLoadBalancerFactory(io.grpc.util.RoundRobinLoadBalancerFactory) SegmentsAtTime(io.pravega.controller.stream.api.grpc.v1.Controller.SegmentsAtTime) VisibleForTesting(com.google.common.annotations.VisibleForTesting) TxnFailedException(io.pravega.client.stream.TxnFailedException) TxnRequest(io.pravega.controller.stream.api.grpc.v1.Controller.TxnRequest) ArrayDeque(java.util.ArrayDeque) SegmentValidityResponse(io.pravega.controller.stream.api.grpc.v1.Controller.SegmentValidityResponse) Collections(java.util.Collections) GrpcSslContexts(io.grpc.netty.GrpcSslContexts) GetSegmentsRequest(io.pravega.controller.stream.api.grpc.v1.Controller.GetSegmentsRequest) StreamInfo(io.pravega.controller.stream.api.grpc.v1.Controller.StreamInfo) SegmentsAtTime(io.pravega.controller.stream.api.grpc.v1.Controller.SegmentsAtTime)

Example 7 with Segment

use of io.pravega.client.segment.impl.Segment in project pravega by pravega.

the class ControllerImpl method getSegmentsInRange.

private Set<Segment> getSegmentsInRange(final StreamCut lowerBound, final Collection<Segment> upperBound) {
    // input validation.
    val fromSCSummary = lowerBound.asImpl().getPositions().keySet().stream().collect(summarizingInt(Segment::getSegmentNumber));
    val toSCSummary = upperBound.stream().collect(summarizingInt(Segment::getSegmentNumber));
    Preconditions.checkArgument(fromSCSummary.getMin() <= toSCSummary.getMin(), "Overlapping StreamCuts cannot be provided");
    Preconditions.checkArgument(fromSCSummary.getMax() <= toSCSummary.getMax(), "Overlapping StreamCuts cannot be provided");
    final HashSet<Segment> segments = new HashSet<>(upperBound);
    segments.addAll(getKnownSegmentsInRange(lowerBound, upperBound));
    ArrayDeque<Segment> toFetchSuccessors = new ArrayDeque<>();
    for (Segment toFetch : lowerBound.asImpl().getPositions().keySet()) {
        if (!segments.contains(toFetch)) {
            toFetchSuccessors.add(toFetch);
        }
    }
    segments.addAll(lowerBound.asImpl().getPositions().keySet());
    while (!toFetchSuccessors.isEmpty()) {
        Segment segment = toFetchSuccessors.remove();
        Set<Segment> successors = getAndHandleExceptions(getSuccessors(segment), RuntimeException::new).getSegmentToPredecessor().keySet();
        for (Segment successor : successors) {
            // Successor segment number can never be larger than the highest segment number in upperBound.
            Preconditions.checkArgument(successor.getSegmentNumber() <= toSCSummary.getMax(), "Overlapping streamCuts, lowerBound streamCut should be strictly lower than the upperBound StreamCut.");
            if (!segments.contains(successor)) {
                segments.add(successor);
                toFetchSuccessors.add(successor);
            }
        }
    }
    return segments;
}
Also used : lombok.val(lombok.val) StatusRuntimeException(io.grpc.StatusRuntimeException) Segment(io.pravega.client.segment.impl.Segment) ArrayDeque(java.util.ArrayDeque) HashSet(java.util.HashSet)

Example 8 with Segment

use of io.pravega.client.segment.impl.Segment in project pravega by pravega.

the class EventStreamReaderImpl method acquireSegmentsIfNeeded.

@GuardedBy("readers")
private void acquireSegmentsIfNeeded() throws ReinitializationRequiredException {
    Map<Segment, Long> newSegments = groupState.acquireNewSegmentsIfNeeded(getLag());
    if (!newSegments.isEmpty()) {
        log.info("{} acquiring segments {}", this, newSegments);
        for (Entry<Segment, Long> newSegment : newSegments.entrySet()) {
            SegmentInputStream in = inputStreamFactory.createInputStreamForSegment(newSegment.getKey());
            in.setOffset(newSegment.getValue());
            readers.add(in);
        }
    }
}
Also used : SegmentInputStream(io.pravega.client.segment.impl.SegmentInputStream) Segment(io.pravega.client.segment.impl.Segment) GuardedBy(javax.annotation.concurrent.GuardedBy)

Example 9 with Segment

use of io.pravega.client.segment.impl.Segment in project pravega by pravega.

the class EventStreamReaderImpl method readNextEvent.

@Override
public EventRead<Type> readNextEvent(long timeout) throws ReinitializationRequiredException, TruncatedDataException {
    synchronized (readers) {
        Preconditions.checkState(!closed, "Reader is closed");
        long waitTime = Math.min(timeout, ReaderGroupStateManager.TIME_UNIT.toMillis());
        Timer timer = new Timer();
        Segment segment = null;
        long offset = -1;
        ByteBuffer buffer;
        do {
            String checkpoint = updateGroupStateIfNeeded();
            if (checkpoint != null) {
                return createEmptyEvent(checkpoint);
            }
            SegmentInputStream segmentReader = orderer.nextSegment(readers);
            if (segmentReader == null) {
                Exceptions.handleInterrupted(() -> Thread.sleep(waitTime));
                buffer = null;
            } else {
                segment = segmentReader.getSegmentId();
                offset = segmentReader.getOffset();
                try {
                    buffer = segmentReader.read(waitTime);
                } catch (EndOfSegmentException e) {
                    handleEndOfSegment(segmentReader);
                    buffer = null;
                } catch (SegmentTruncatedException e) {
                    handleSegmentTruncated(segmentReader);
                    buffer = null;
                }
            }
        } while (buffer == null && timer.getElapsedMillis() < timeout);
        if (buffer == null) {
            return createEmptyEvent(null);
        }
        lastRead = Sequence.create(segment.getSegmentNumber(), offset);
        int length = buffer.remaining() + WireCommands.TYPE_PLUS_LENGTH_SIZE;
        return new EventReadImpl<>(lastRead, deserializer.deserialize(buffer), getPosition(), new EventPointerImpl(segment, offset, length), null);
    }
}
Also used : SegmentInputStream(io.pravega.client.segment.impl.SegmentInputStream) Timer(io.pravega.common.Timer) EndOfSegmentException(io.pravega.client.segment.impl.EndOfSegmentException) SegmentTruncatedException(io.pravega.client.segment.impl.SegmentTruncatedException) ByteBuffer(java.nio.ByteBuffer) Segment(io.pravega.client.segment.impl.Segment)

Example 10 with Segment

use of io.pravega.client.segment.impl.Segment in project pravega by pravega.

the class EventStreamReaderImpl method handleSegmentTruncated.

private void handleSegmentTruncated(SegmentInputStream segmentReader) throws ReinitializationRequiredException, TruncatedDataException {
    Segment segmentId = segmentReader.getSegmentId();
    log.info("{} encountered truncation for segment {} ", this, segmentId);
    String delegationToken = groupState.getLatestDelegationToken();
    @Cleanup SegmentMetadataClient metadataClient = metadataClientFactory.createSegmentMetadataClient(segmentId, delegationToken);
    try {
        long startingOffset = metadataClient.getSegmentInfo().getStartingOffset();
        segmentReader.setOffset(startingOffset);
    } catch (NoSuchSegmentException e) {
        handleEndOfSegment(segmentReader);
    }
    throw new TruncatedDataException();
}
Also used : TruncatedDataException(io.pravega.client.stream.TruncatedDataException) Cleanup(lombok.Cleanup) Segment(io.pravega.client.segment.impl.Segment) SegmentMetadataClient(io.pravega.client.segment.impl.SegmentMetadataClient) NoSuchSegmentException(io.pravega.client.segment.impl.NoSuchSegmentException)

Aggregations

Segment (io.pravega.client.segment.impl.Segment)97 Test (org.junit.Test)63 Cleanup (lombok.Cleanup)35 HashMap (java.util.HashMap)27 MockSegmentStreamFactory (io.pravega.client.stream.mock.MockSegmentStreamFactory)22 SegmentOutputStream (io.pravega.client.segment.impl.SegmentOutputStream)19 AtomicLong (java.util.concurrent.atomic.AtomicLong)19 PravegaNodeUri (io.pravega.shared.protocol.netty.PravegaNodeUri)17 SegmentOutputStreamFactory (io.pravega.client.segment.impl.SegmentOutputStreamFactory)16 EventWriterConfig (io.pravega.client.stream.EventWriterConfig)16 MockController (io.pravega.client.stream.mock.MockController)15 ClientFactory (io.pravega.client.ClientFactory)13 Stream (io.pravega.client.stream.Stream)13 MockConnectionFactoryImpl (io.pravega.client.stream.mock.MockConnectionFactoryImpl)13 SynchronizerConfig (io.pravega.client.state.SynchronizerConfig)12 InlineExecutor (io.pravega.test.common.InlineExecutor)12 TreeMap (java.util.TreeMap)10 lombok.val (lombok.val)10 StreamCut (io.pravega.client.stream.StreamCut)8 ByteBuffer (java.nio.ByteBuffer)8