Examples with EventWriterConfig io.pravega.client.stream.EventWriterConfig used on opensource projects

Example 31 with EventWriterConfig

use of io.pravega.client.stream.EventWriterConfig in project pravega by pravega.

the class EndToEndTransactionOrderTest method startWriter.

private CompletableFuture<Void> startWriter(String writerId, MockClientFactory clientFactory, AtomicBoolean done) {
    EventWriterConfig writerConfig = EventWriterConfig.builder().transactionTimeoutTime(30000).build();
    TransactionalEventStreamWriter<Integer> test = clientFactory.createTransactionalEventWriter(writerId, "test", new IntegerSerializer(), writerConfig);
    List<UUID> list = new LinkedList<>();
    writersList.put(writerId, list);
    // Mocking pravega service by putting scale up and scale down requests for the stream
    return Futures.loop(() -> !done.get(), () -> Futures.delayedFuture(Duration.ofMillis(10), executor).thenAccept(v -> {
        try {
            Transaction<Integer> transaction = test.beginTxn();
            transaction.getTxnId();
            int i1 = counter.incrementAndGet();
            transaction.writeEvent("0", i1);
            transaction.commit();
            list.add(transaction.getTxnId());
            eventToTxnMap.put(i1, transaction.getTxnId());
            txnToWriter.put(transaction.getTxnId(), writerId);
        } catch (Throwable e) {
            log.error("test exception writing events", e);
            throw new CompletionException(e);
        }
    }), executor).thenAccept(v -> test.close());
}

Example 32 with EventWriterConfig

use of io.pravega.client.stream.EventWriterConfig in project pravega by pravega.

the class EndToEndTxnWithTest method testGetTxnWithScale.

@Test(timeout = 20000)
public void testGetTxnWithScale() throws Exception {
    String streamName = "testGetTxnWithScale";
    final StreamConfiguration config = StreamConfiguration.builder().scalingPolicy(ScalingPolicy.fixed(1)).build();
    final Serializer<String> serializer = new UTF8StringSerializer();
    final EventWriterConfig writerConfig = EventWriterConfig.builder().transactionTimeoutTime(10000).build();
    final Controller controller = PRAVEGA.getLocalController();
    controller.createScope("test").get();
    controller.createStream("test", streamName, config).get();
    @Cleanup ConnectionFactory connectionFactory = new SocketConnectionFactoryImpl(ClientConfig.builder().build());
    @Cleanup ClientFactoryImpl clientFactory = new ClientFactoryImpl("test", controller, connectionFactory);
    @Cleanup EventStreamWriter<String> streamWriter = clientFactory.createEventWriter(streamName, serializer, writerConfig);
    streamWriter.writeEvent("key", "e").join();
    @Cleanup TransactionalEventStreamWriter<String> txnWriter = clientFactory.createTransactionalEventWriter(streamName, serializer, writerConfig);
    Transaction<String> txn = txnWriter.beginTxn();
    txn.writeEvent("key", "1");
    txn.flush();
    // the txn is not yet committed here.
    UUID txnId = txn.getTxnId();
    // scale up stream
    scaleUpStream(streamName);
    // write event using stream writer
    streamWriter.writeEvent("key", "e").join();
    Transaction<String> txn1 = txnWriter.getTxn(txnId);
    txn1.writeEvent("key", "2");
    txn1.flush();
    // commit the transaction
    txn1.commit();
    assertEventuallyEquals(Transaction.Status.COMMITTED, txn1::checkStatus, 5000);
    String group = "testGetTxnWithScale-group";
    @Cleanup ReaderGroupManager groupManager = new ReaderGroupManagerImpl("test", controller, clientFactory);
    groupManager.createReaderGroup(group, ReaderGroupConfig.builder().disableAutomaticCheckpoints().groupRefreshTimeMillis(0).stream("test/" + streamName).build());
    @Cleanup EventStreamReader<String> reader = clientFactory.createReader("readerId", group, new UTF8StringSerializer(), ReaderConfig.builder().build());
    EventRead<String> event = reader.readNextEvent(5000);
    assertEquals("e", event.getEvent());
    assertNull(reader.readNextEvent(100).getEvent());
    groupManager.getReaderGroup(group).initiateCheckpoint("cp1", executorService());
    event = reader.readNextEvent(5000);
    assertEquals("Checkpoint event expected", "cp1", event.getCheckpointName());
    event = reader.readNextEvent(5000);
    assertEquals("second event post scale up", "e", event.getEvent());
    assertNull(reader.readNextEvent(100).getEvent());
    groupManager.getReaderGroup(group).initiateCheckpoint("cp2", executorService());
    event = reader.readNextEvent(5000);
    assertEquals("Checkpoint event expected", "cp2", event.getCheckpointName());
    event = reader.readNextEvent(5000);
    assertEquals("txn events", "1", event.getEvent());
    event = reader.readNextEvent(5000);
    assertEquals("txn events", "2", event.getEvent());
}

Example 33 with EventWriterConfig

use of io.pravega.client.stream.EventWriterConfig in project pravega by pravega.

the class EndToEndTxnWithTest method testTxnConfig.

@Test(timeout = 10000)
public void testTxnConfig() throws Exception {
    // create stream test
    StreamConfiguration config = StreamConfiguration.builder().scalingPolicy(ScalingPolicy.byEventRate(10, 2, 1)).build();
    Controller controller = PRAVEGA.getLocalController();
    controller.createScope("test").get();
    String streamName = "testTxnConfig";
    controller.createStream("test", streamName, config).get();
    @Cleanup ConnectionFactory connectionFactory = new SocketConnectionFactoryImpl(ClientConfig.builder().build());
    @Cleanup ClientFactoryImpl clientFactory = new ClientFactoryImpl("test", controller, connectionFactory);
    // create writers with different configs and try creating transactions against those configs
    EventWriterConfig defaultConfig = EventWriterConfig.builder().build();
    assertNotNull(createTxn(clientFactory, defaultConfig, streamName));
    EventWriterConfig validConfig = EventWriterConfig.builder().transactionTimeoutTime(10000).build();
    assertNotNull(createTxn(clientFactory, validConfig, streamName));
    AssertExtensions.assertThrows("low timeout period not honoured", () -> {
        EventWriterConfig lowTimeoutConfig = EventWriterConfig.builder().transactionTimeoutTime(1000).build();
        createTxn(clientFactory, lowTimeoutConfig, streamName);
    }, e -> Exceptions.unwrap(e) instanceof IllegalArgumentException);
    EventWriterConfig highTimeoutConfig = EventWriterConfig.builder().transactionTimeoutTime(700 * 1000).build();
    AssertExtensions.assertThrows("lease value too large, max value is 600000", () -> createTxn(clientFactory, highTimeoutConfig, streamName), e -> e instanceof IllegalArgumentException);
}

Example 34 with EventWriterConfig

use of io.pravega.client.stream.EventWriterConfig in project pravega by pravega.

the class EventStreamWriterTest method testTxn.

@Test
public void testTxn() throws TxnFailedException {
    String scope = "scope";
    String streamName = "stream";
    StreamImpl stream = new StreamImpl(scope, streamName);
    Segment segment = new Segment(scope, streamName, 0);
    UUID txid = UUID.randomUUID();
    EventWriterConfig config = EventWriterConfig.builder().transactionTimeoutTime(0).transactionTimeoutScaleGracePeriod(0).build();
    SegmentOutputStreamFactory streamFactory = Mockito.mock(SegmentOutputStreamFactory.class);
    Controller controller = Mockito.mock(Controller.class);
    Mockito.when(controller.getCurrentSegments(scope, streamName)).thenReturn(getSegmentsFuture(segment));
    FakeSegmentOutputStream outputStream = new FakeSegmentOutputStream(segment);
    FakeSegmentOutputStream bad = new FakeSegmentOutputStream(segment);
    Mockito.when(controller.createTransaction(stream, 0, 0)).thenReturn(CompletableFuture.completedFuture(new TxnSegments(getSegments(segment), txid)));
    Mockito.when(streamFactory.createOutputStreamForTransaction(eq(segment), eq(txid), any(), any(), any())).thenReturn(outputStream);
    Mockito.when(streamFactory.createOutputStreamForSegment(eq(segment), any(), any(), any())).thenReturn(bad);
    JavaSerializer<String> serializer = new JavaSerializer<>();
    @Cleanup EventStreamWriter<String> writer = new EventStreamWriterImpl<>(stream, controller, streamFactory, serializer, config, new InlineExecutor());
    Transaction<String> txn = writer.beginTxn();
    txn.writeEvent("Foo");
    Mockito.verify(controller).getCurrentSegments(any(), any());
    assertTrue(bad.getUnackedEventsOnSeal().isEmpty());
    assertEquals(1, outputStream.getUnackedEventsOnSeal().size());
    outputStream.getUnackedEventsOnSeal().get(0).getAckFuture().complete(true);
    txn.flush();
    assertTrue(bad.getUnackedEventsOnSeal().isEmpty());
    assertTrue(outputStream.getUnackedEventsOnSeal().isEmpty());
}

Example 35 with EventWriterConfig

use of io.pravega.client.stream.EventWriterConfig in project pravega by pravega.

the class LargeEventWriter method writeLargeEvent.

/**
 * Write the provided list of events (atomically) to the provided segment.
 *
 * @param segment The segment to write to
 * @param events The events to append
 * @param tokenProvider A token provider
 * @param config Used for retry configuration parameters
 * @throws NoSuchSegmentException If the provided segment does not exit.
 * @throws SegmentSealedException If the segment is sealed.
 * @throws AuthenticationException If the token can't be used for this segment.
 * @throws UnsupportedOperationException If the server does not support large events.
 */
public void writeLargeEvent(Segment segment, List<ByteBuffer> events, DelegationTokenProvider tokenProvider, EventWriterConfig config) throws NoSuchSegmentException, AuthenticationException, SegmentSealedException {
    List<ByteBuf> payloads = createBufs(events);
    int attempts = 1 + Math.max(0, config.getRetryAttempts());
    Retry.withExpBackoff(config.getInitialBackoffMillis(), config.getBackoffMultiple(), attempts, config.getMaxBackoffMillis()).retryWhen(t -> {
        Throwable ex = Exceptions.unwrap(t);
        if (ex instanceof ConnectionFailedException) {
            log.info("Connection failure while sending large event: {}. Retrying", ex.getMessage());
            return true;
        } else if (ex instanceof TokenExpiredException) {
            tokenProvider.signalTokenExpired();
            log.info("Authentication token expired while writing large event to segment {}. Retrying", segment);
            return true;
        } else {
            return false;
        }
    }).run(() -> {
        @Cleanup RawClient client = new RawClient(controller, connectionPool, segment);
        write(segment, payloads, client, tokenProvider);
        return null;
    });
}