Examples with SegmentOutputStream io.pravega.client.segment.impl.SegmentOutputStream used on opensource projects

Example 1 with SegmentOutputStream

use of io.pravega.client.segment.impl.SegmentOutputStream in project pravega by pravega.

the class ClientFactoryImpl method createRevisionedStreamClient.

@Override
public <T> RevisionedStreamClient<T> createRevisionedStreamClient(String streamName, Serializer<T> serializer, SynchronizerConfig config) {
    log.info("Creating revisioned stream client for stream: {} with synchronizer configuration: {}", streamName, config);
    Segment segment = new Segment(scope, streamName, 0);
    SegmentInputStream in = inFactory.createInputStreamForSegment(segment);
    // Segment sealed is not expected for Revisioned Stream Client.
    Consumer<Segment> segmentSealedCallBack = s -> {
        throw new IllegalStateException("RevisionedClient: Segmentsealed exception observed for segment:" + s);
    };
    String delegationToken = Futures.getAndHandleExceptions(controller.getOrRefreshDelegationTokenFor(segment.getScope(), segment.getStreamName()), RuntimeException::new);
    SegmentOutputStream out = outFactory.createOutputStreamForSegment(segment, segmentSealedCallBack, config.getEventWriterConfig(), delegationToken);
    SegmentMetadataClient meta = metaFactory.createSegmentMetadataClient(segment, delegationToken);
    return new RevisionedStreamClientImpl<>(segment, in, out, meta, serializer, controller, delegationToken);
}

Example 2 with SegmentOutputStream

use of io.pravega.client.segment.impl.SegmentOutputStream in project pravega by pravega.

the class EventStreamWriterImpl method beginTxn.

@Override
public Transaction<Type> beginTxn() {
    TxnSegments txnSegments = getAndHandleExceptions(controller.createTransaction(stream, config.getTransactionTimeoutTime(), config.getTransactionTimeoutScaleGracePeriod()), RuntimeException::new);
    UUID txnId = txnSegments.getTxnId();
    Map<Segment, SegmentTransaction<Type>> transactions = new HashMap<>();
    for (Segment s : txnSegments.getSteamSegments().getSegments()) {
        SegmentOutputStream out = outputStreamFactory.createOutputStreamForTransaction(s, txnId, segmentSealedCallBack, config, txnSegments.getSteamSegments().getDelegationToken());
        SegmentTransactionImpl<Type> impl = new SegmentTransactionImpl<>(txnId, out, serializer);
        transactions.put(s, impl);
    }
    pinger.startPing(txnId);
    return new TransactionImpl<Type>(txnId, transactions, txnSegments.getSteamSegments(), controller, stream, pinger);
}

Example 3 with SegmentOutputStream

use of io.pravega.client.segment.impl.SegmentOutputStream in project pravega by pravega.

the class EventStreamWriterImpl method getTxn.

@Override
public Transaction<Type> getTxn(UUID txId) {
    StreamSegments segments = getAndHandleExceptions(controller.getCurrentSegments(stream.getScope(), stream.getStreamName()), RuntimeException::new);
    Status status = getAndHandleExceptions(controller.checkTransactionStatus(stream, txId), RuntimeException::new);
    if (status != Status.OPEN) {
        return new TransactionImpl<>(txId, controller, stream);
    }
    Map<Segment, SegmentTransaction<Type>> transactions = new HashMap<>();
    for (Segment s : segments.getSegments()) {
        SegmentOutputStream out = outputStreamFactory.createOutputStreamForTransaction(s, txId, segmentSealedCallBack, config, segments.getDelegationToken());
        SegmentTransactionImpl<Type> impl = new SegmentTransactionImpl<>(txId, out, serializer);
        transactions.put(s, impl);
    }
    return new TransactionImpl<Type>(txId, transactions, segments, controller, stream, pinger);
}

Example 4 with SegmentOutputStream

use of io.pravega.client.segment.impl.SegmentOutputStream in project pravega by pravega.

the class EventStreamWriterImpl method writeEvents.

@Override
public CompletableFuture<Void> writeEvents(String routingKey, List<Type> events) {
    Preconditions.checkNotNull(routingKey);
    Preconditions.checkNotNull(events);
    Exceptions.checkNotClosed(closed.get(), this);
    List<ByteBuffer> data = events.stream().map(serializer::serialize).collect(Collectors.toList());
    CompletableFuture<Void> ackFuture = new CompletableFuture<Void>();
    synchronized (writeFlushLock) {
        if (config.isEnableLargeEvents() && data.stream().mapToInt(m -> m.remaining()).sum() > Serializer.MAX_EVENT_SIZE) {
            writeLargeEvent(routingKey, data, ackFuture);
        } else {
            synchronized (writeSealLock) {
                SegmentOutputStream segmentWriter = getSegmentWriter(routingKey);
                segmentWriter.write(PendingEvent.withHeader(routingKey, data, ackFuture));
            }
        }
    }
    return ackFuture;
}

Example 5 with SegmentOutputStream

use of io.pravega.client.segment.impl.SegmentOutputStream in project pravega by pravega.

the class EventStreamWriterImpl method handleLogSealed.

/**
 * If a log sealed is encountered, we need to: 1. Find the new segments to write to. 2. For each outstanding
 * message find which new segment it should go to and send it there.
 */
private void handleLogSealed(Segment segment) {
    sealedSegmentQueueEmptyLatch.reset();
    sealedSegmentQueue.add(segment);
    retransmitPool.execute(() -> {
        Retry.indefinitelyWithExpBackoff(config.getInitialBackoffMillis(), config.getBackoffMultiple(), config.getMaxBackoffMillis(), t -> log.error("Encountered exception when handling a sealed segment: ", t)).run(() -> {
            /*
                      * Using writeSealLock prevents concurrent segmentSealedCallback for different segments
                      * from being invoked concurrently, or concurrently with write.
                      * 
                      * By calling flush while the write lock is held we can ensure that any inflight
                      * entries that will succeed in being written to a new segment are written and any
                      * segmentSealedCallbacks that will be called happen before the next write is invoked.
                      */
            synchronized (writeSealLock) {
                Segment toSeal = sealedSegmentQueue.poll();
                log.info("Sealing segment {} ", toSeal);
                while (toSeal != null) {
                    resend(selector.refreshSegmentEventWritersUponSealed(toSeal, segmentSealedCallBack));
                    // remove segment writer after resending inflight events of the sealed segment.
                    selector.removeSegmentWriter(toSeal);
                    /* In the case of segments merging Flush ensures there can't be anything left
                              * inflight that will need to be resent to the new segment when the write lock
                              * is released. (To preserve order)
                              */
                    for (SegmentOutputStream writer : selector.getWriters().values()) {
                        try {
                            writer.write(PendingEvent.withoutHeader(null, ByteBufferUtils.EMPTY, null));
                            writer.flush();
                        } catch (SegmentSealedException e) {
                            // Segment sealed exception observed during a flush. Re-run flush on all the
                            // available writers.
                            log.info("Flush on segment {} failed due to {}, it will be retried.", writer.getSegmentName(), e.getMessage());
                        } catch (RetriesExhaustedException e1) {
                            log.warn("Flush on segment {} failed after all retries", writer.getSegmentName(), e1);
                        }
                    }
                    toSeal = sealedSegmentQueue.poll();
                    if (toSeal != null) {
                        log.info("Sealing another segment {} ", toSeal);
                    }
                }
                sealedSegmentQueueEmptyLatch.release();
            }
            return null;
        });
    });
}