Examples with UTF8StringSerializer io.pravega.client.stream.impl.UTF8StringSerializer used on opensource projects

Example 1 with UTF8StringSerializer

use of io.pravega.client.stream.impl.UTF8StringSerializer in project pravega by pravega.

the class TestUtils method readAllEvents.

/**
 * Read all events from the given stream.
 *
 * @param scope               Scope of the targeted Stream.
 * @param streamName          Name of the Stream.
 * @param clientFactory       ClientFactory to instantiate readers.
 * @param readerGroupManager  ReaderGroupManager to create the ReaderGroup.
 * @param readerGroupName     Name of the ReadeGroup to be created.
 * @param readerName          Name of the Reader to instantiate.
 */
public static void readAllEvents(String scope, String streamName, ClientFactoryImpl clientFactory, ReaderGroupManager readerGroupManager, String readerGroupName, String readerName) {
    readerGroupManager.createReaderGroup(readerGroupName, ReaderGroupConfig.builder().stream(Stream.of(scope, streamName)).build());
    @Cleanup EventStreamReader<String> reader = clientFactory.createReader(readerName, readerGroupName, new UTF8StringSerializer(), ReaderConfig.builder().build());
    for (int q = 0; q < NUM_EVENTS; q++) {
        String eventRead = reader.readNextEvent(READ_TIMEOUT.toMillis()).getEvent();
        Assert.assertEquals("Event written and read back don't match", EVENT, eventRead);
    }
}

Example 2 with UTF8StringSerializer

use of io.pravega.client.stream.impl.UTF8StringSerializer in project pravega by pravega.

the class TestUtils method writeEvents.

/**
 * Write events to the given stream.
 *
 * @param streamName     Name of the Stream.
 * @param clientFactory  Client factory to create writers.
 */
public static void writeEvents(String streamName, ClientFactoryImpl clientFactory) {
    @Cleanup EventStreamWriter<String> writer = clientFactory.createEventWriter(streamName, new UTF8StringSerializer(), EventWriterConfig.builder().build());
    for (int i = 0; i < NUM_EVENTS; i++) {
        writer.writeEvent(EVENT).join();
    }
    writer.flush();
}

Example 3 with UTF8StringSerializer

use of io.pravega.client.stream.impl.UTF8StringSerializer in project pravega by pravega.

the class EndToEndAutoScaleUpWithTxnTest method main.

public static void main(String[] args) throws Exception {
    try {
        @Cleanup TestingServer zkTestServer = new TestingServerStarter().start();
        int port = Config.SERVICE_PORT;
        @Cleanup ControllerWrapper controllerWrapper = new ControllerWrapper(zkTestServer.getConnectString(), port);
        Controller controller = controllerWrapper.getController();
        controllerWrapper.getControllerService().createScope(NameUtils.INTERNAL_SCOPE_NAME, 0L).get();
        @Cleanup ConnectionFactory connectionFactory = new SocketConnectionFactoryImpl(ClientConfig.builder().build());
        @Cleanup ClientFactoryImpl internalCF = new ClientFactoryImpl(NameUtils.INTERNAL_SCOPE_NAME, controller, connectionFactory);
        @Cleanup("shutdownNow") val executor = ExecutorServiceHelpers.newScheduledThreadPool(1, "test");
        @Cleanup ServiceBuilder serviceBuilder = ServiceBuilder.newInMemoryBuilder(ServiceBuilderConfig.getDefaultConfig());
        serviceBuilder.initialize();
        StreamSegmentStore store = serviceBuilder.createStreamSegmentService();
        TableStore tableStore = serviceBuilder.createTableStoreService();
        @Cleanup AutoScaleMonitor autoScaleMonitor = new AutoScaleMonitor(store, internalCF, AutoScalerConfig.builder().with(AutoScalerConfig.MUTE_IN_SECONDS, 0).with(AutoScalerConfig.COOLDOWN_IN_SECONDS, 0).build());
        @Cleanup PravegaConnectionListener server = new PravegaConnectionListener(false, false, "localhost", 12345, store, tableStore, autoScaleMonitor.getStatsRecorder(), autoScaleMonitor.getTableSegmentStatsRecorder(), null, null, null, true, serviceBuilder.getLowPriorityExecutor(), Config.TLS_PROTOCOL_VERSION.toArray(new String[Config.TLS_PROTOCOL_VERSION.size()]));
        server.startListening();
        controllerWrapper.awaitRunning();
        controllerWrapper.getControllerService().createScope("test", 0L).get();
        controller.createStream("test", "test", CONFIG).get();
        @Cleanup MockClientFactory clientFactory = new MockClientFactory("test", controller, internalCF.getConnectionPool());
        // Mocking pravega service by putting scale up and scale down requests for the stream
        EventWriterConfig writerConfig = EventWriterConfig.builder().transactionTimeoutTime(30000).build();
        TransactionalEventStreamWriter<String> test = clientFactory.createTransactionalEventWriter("writer", "test", new UTF8StringSerializer(), writerConfig);
        // region Successful commit tests
        Transaction<String> txn1 = test.beginTxn();
        txn1.writeEvent("1");
        txn1.flush();
        Map<Double, Double> map = new HashMap<>();
        map.put(0.0, 1.0 / 3.0);
        map.put(1.0 / 3.0, 2.0 / 3.0);
        map.put(2.0 / 3.0, 1.0);
        Stream stream = new StreamImpl("test", "test");
        controller.startScale(stream, Collections.singletonList(0L), map).get();
        Transaction<String> txn2 = test.beginTxn();
        txn2.writeEvent("2");
        txn2.flush();
        txn2.commit();
        txn1.commit();
        Thread.sleep(1000);
        @Cleanup ReaderGroupManager readerGroupManager = new ReaderGroupManagerImpl("test", controller, clientFactory);
        readerGroupManager.createReaderGroup("readergrp", ReaderGroupConfig.builder().stream("test/test").build());
        final EventStreamReader<String> reader = clientFactory.createReader("1", "readergrp", new JavaSerializer<>(), ReaderConfig.builder().build());
        String event1 = reader.readNextEvent(SECONDS.toMillis(60)).getEvent();
        String event2 = reader.readNextEvent(SECONDS.toMillis(60)).getEvent();
        assert event1.equals("1");
        assert event2.equals("2");
        final AtomicBoolean done = new AtomicBoolean(false);
        startWriter(test, done);
        Retry.withExpBackoff(10, 10, 100, 10000).retryingOn(NotDoneException.class).throwingOn(RuntimeException.class).runAsync(() -> controller.getCurrentSegments("test", "test").thenAccept(streamSegments -> {
            if (streamSegments.getSegments().stream().anyMatch(x -> NameUtils.getEpoch(x.getSegmentId()) > 5)) {
                System.err.println("Success");
                log.info("Success");
                System.exit(0);
            } else {
                throw new NotDoneException();
            }
        }), executor).exceptionally(e -> {
            System.err.println("Failure");
            log.error("Failure");
            System.exit(1);
            return null;
        }).get();
    } catch (Throwable e) {
        System.err.print("Test failed with exception: " + e.getMessage());
        log.error("Test failed with exception: {}", e);
        System.exit(-1);
    }
    System.exit(0);
}

Example 4 with UTF8StringSerializer

use of io.pravega.client.stream.impl.UTF8StringSerializer in project pravega by pravega.

the class RestoreBackUpDataRecoveryTest method testDurableDataLogFailRecoveryReadersPaused.

/**
 * Tests the data recovery scenario with readers stalling while reading. Readers read some events and then they are
 * stopped. Durable data log is erased and restored. It's validated that readers are able to read rest of the unread
 * events.
 *  What test does, step by step:
 *  1. Starts Pravega locally with just 4 segment containers.
 *  2. Writes {@link #TOTAL_NUM_EVENTS} events.
 *  3. Waits for all segments created to be flushed to the long term storage.
 *  4. Let a reader read N number of events.
 *  5. Shuts down the controller, segment store and bookeeper/zookeeper.
 *  6. Creates back up of container metadata segment and its attribute segment before deleting them from the Long Term Storage .
 *  7. Starts 4 debug segment containers using a new bookeeper/zookeeper and the Long Term Storage.
 *  8. Re-creates the container metadata segments in DurableLog and lets them to be flushed to the Long Term Storage.
 *  9. Updates core attributes of segments in the new container metadata segment by using details from the back up of old container metadata segment.
 *  10. Starts segment store and controller.
 *  11. Let the reader read rest of the 10-N number of events.
 * @throws Exception    In case of an exception occurred while execution.
 */
@Test(timeout = 180000)
public void testDurableDataLogFailRecoveryReadersPaused() throws Exception {
    int instanceId = 0;
    int bookieCount = 1;
    int containerCount = 4;
    int eventsReadCount = RANDOM.nextInt(TOTAL_NUM_EVENTS);
    String testReader = "readerDRIntegrationTest";
    String testReaderGroup = "readerGroupDRIntegrationTest";
    // Creating a long term storage only once here.
    this.storageFactory = new InMemoryStorageFactory(executorService());
    log.info("Created a long term storage.");
    // Start a new BK & ZK, segment store and controller
    @Cleanup PravegaRunner pravegaRunner = new PravegaRunner(instanceId++, bookieCount, containerCount, this.storageFactory);
    // Create a stream for writing data
    createScopeStream(pravegaRunner.controllerRunner.controller, SCOPE, STREAM1);
    log.info("Created stream '{}'", STREAM1);
    // Create a client to write events.
    try (val clientRunner = new ClientRunner(pravegaRunner.controllerRunner)) {
        // Write events.
        writeEventsToStream(clientRunner.clientFactory, true);
        // Create a reader for reading from the stream.
        EventStreamReader<String> reader = createReader(clientRunner.clientFactory, clientRunner.readerGroupManager, SCOPE, STREAM1, testReaderGroup, testReader);
        // Let reader read N number of events and mark its position.
        Position p = readNEvents(reader, eventsReadCount);
        ReaderGroup readerGroup = clientRunner.readerGroupManager.getReaderGroup(testReaderGroup);
        readerGroup.readerOffline(testReader, p);
    }
    // Shut down the controller
    pravegaRunner.controllerRunner.close();
    // Flush DurableLog to Long Term Storage
    flushToStorage(pravegaRunner.segmentStoreRunner.serviceBuilder);
    // Shutdown SegmentStore
    pravegaRunner.segmentStoreRunner.close();
    // Shutdown BookKeeper & ZooKeeper
    pravegaRunner.bookKeeperRunner.close();
    log.info("SegmentStore, BookKeeper & ZooKeeper shutdown");
    // Get the long term storage from the running pravega instance
    @Cleanup Storage storage = new AsyncStorageWrapper(new RollingStorage(this.storageFactory.createSyncStorage(), new SegmentRollingPolicy(DEFAULT_ROLLING_SIZE)), executorService());
    Map<Integer, String> backUpMetadataSegments = ContainerRecoveryUtils.createBackUpMetadataSegments(storage, containerCount, executorService(), TIMEOUT).join();
    // start a new BookKeeper and ZooKeeper.
    pravegaRunner.bookKeeperRunner = new BookKeeperRunner(instanceId++, bookieCount);
    createBookKeeperLogFactory();
    log.info("Started a new BookKeeper and ZooKeeper.");
    // Recover segments
    runRecovery(containerCount, storage, backUpMetadataSegments);
    // Start a new segment store and controller
    pravegaRunner.restartControllerAndSegmentStore(this.storageFactory, this.dataLogFactory);
    log.info("Started segment store and controller again.");
    // Create the client with new controller.
    try (val clientRunner = new ClientRunner(pravegaRunner.controllerRunner)) {
        // Get reader group.
        ReaderGroup readerGroup = clientRunner.readerGroupManager.getReaderGroup(testReaderGroup);
        assertNotNull(readerGroup);
        EventStreamReader<String> reader = clientRunner.clientFactory.createReader(testReader, testReaderGroup, new UTF8StringSerializer(), ReaderConfig.builder().build());
        // Read the remaining number of events.
        readNEvents(reader, TOTAL_NUM_EVENTS - eventsReadCount);
        // Reading next event should return null.
        assertNull(reader.readNextEvent(READ_TIMEOUT.toMillis()).getEvent());
        reader.close();
    }
}

Example 5 with UTF8StringSerializer

use of io.pravega.client.stream.impl.UTF8StringSerializer in project pravega by pravega.

the class RestoreBackUpDataRecoveryTest method writeTransactionalEvents.

// Writes the required number of events to the given stream with using transactions.
private void writeTransactionalEvents(String streamName, ClientFactoryImpl clientFactory) throws TxnFailedException {
    EventWriterConfig writerConfig = EventWriterConfig.builder().transactionTimeoutTime(TRANSACTION_TIMEOUT.toMillis()).build();
    @Cleanup TransactionalEventStreamWriter<String> txnWriter = clientFactory.createTransactionalEventWriter(streamName, new UTF8StringSerializer(), writerConfig);
    Transaction<String> transaction = txnWriter.beginTxn();
    for (int i = 0; i < TOTAL_NUM_EVENTS; i++) {
        transaction.writeEvent("0", EVENT);
    }
    transaction.commit();
    txnWriter.close();
}