Examples with TableStore io.pravega.segmentstore.contracts.tables.TableStore used on opensource projects

Example 56 with TableStore

use of io.pravega.segmentstore.contracts.tables.TableStore in project pravega by pravega.

the class PravegaRequestProcessorTest method testReadTableEntriesDeltaOutOfBounds.

@Test
public void testReadTableEntriesDeltaOutOfBounds() throws Exception {
    // Set up PravegaRequestProcessor instance to execute requests against
    val rnd = new Random(0);
    String tableSegmentName = "testReadTableEntriesDelta";
    @Cleanup ServiceBuilder serviceBuilder = newInlineExecutionInMemoryBuilder(getBuilderConfig());
    serviceBuilder.initialize();
    StreamSegmentStore store = serviceBuilder.createStreamSegmentService();
    TableStore tableStore = serviceBuilder.createTableStoreService();
    ServerConnection connection = mock(ServerConnection.class);
    InOrder order = inOrder(connection);
    val recorderMock = mock(TableSegmentStatsRecorder.class);
    PravegaRequestProcessor processor = new PravegaRequestProcessor(store, tableStore, new TrackedConnection(connection), SegmentStatsRecorder.noOp(), recorderMock, new PassingTokenVerifier(), false);
    processor.createTableSegment(new WireCommands.CreateTableSegment(1, tableSegmentName, false, 0, "", 0));
    order.verify(connection).send(new WireCommands.SegmentCreated(1, tableSegmentName));
    verify(recorderMock).createTableSegment(eq(tableSegmentName), any());
    // Generate keys.
    ArrayList<ArrayView> keys = generateKeys(3, rnd);
    ArrayView testValue = generateValue(rnd);
    TableEntry e1 = TableEntry.unversioned(keys.get(0), testValue);
    TableEntry e2 = TableEntry.unversioned(keys.get(1), testValue);
    TableEntry e3 = TableEntry.unversioned(keys.get(2), testValue);
    processor.updateTableEntries(new WireCommands.UpdateTableEntries(2, tableSegmentName, "", getTableEntries(asList(e1, e2, e3)), WireCommands.NULL_TABLE_SEGMENT_OFFSET));
    verify(recorderMock).updateEntries(eq(tableSegmentName), eq(3), eq(false), any());
    long length = store.getStreamSegmentInfo(tableSegmentName, Duration.ofMinutes(1)).get().getLength();
    processor.readTableEntriesDelta(new WireCommands.ReadTableEntriesDelta(1, tableSegmentName, "", length + 1, 3));
    order.verify(connection).send(new WireCommands.ErrorMessage(1, tableSegmentName, "fromPosition (652) can not exceed the length (651) of the TableSegment.", WireCommands.ErrorMessage.ErrorCode.valueOf(IllegalArgumentException.class)));
    verify(recorderMock, times(0)).iterateEntries(eq(tableSegmentName), eq(3), any());
}

Example 57 with TableStore

use of io.pravega.segmentstore.contracts.tables.TableStore in project pravega by pravega.

the class SetupUtils method startAllServices.

/**
 * Start all pravega related services required for the test deployment.
 *
 * @param numThreads the number of threads for the internal client threadpool.
 * @param enableAuth set to enale authentication
 * @param enableTls set to enable tls
 * @throws Exception on any errors.
 */
public void startAllServices(Integer numThreads, boolean enableAuth, boolean enableTls) throws Exception {
    if (!this.started.compareAndSet(false, true)) {
        log.warn("Services already started, not attempting to start again");
        return;
    }
    if (enableAuth) {
        clientConfigBuilder = clientConfigBuilder.credentials(new DefaultCredentials(SecurityConfigDefaults.AUTH_ADMIN_PASSWORD, SecurityConfigDefaults.AUTH_ADMIN_USERNAME));
    }
    if (enableTls) {
        clientConfigBuilder = clientConfigBuilder.trustStore(pathToConfig() + SecurityConfigDefaults.TLS_CA_CERT_FILE_NAME).controllerURI(URI.create("tls://localhost:" + controllerRPCPort)).validateHostName(false);
    } else {
        clientConfigBuilder = clientConfigBuilder.controllerURI(URI.create("tcp://localhost:" + controllerRPCPort));
    }
    this.executor = ExecutorServiceHelpers.newScheduledThreadPool(2, "Controller pool");
    this.controller = new ControllerImpl(ControllerImplConfig.builder().clientConfig(getClientConfig()).build(), executor);
    this.clientFactory = new ClientFactoryImpl(scope, controller, getClientConfig());
    // Start zookeeper.
    this.zkTestServer = new TestingServerStarter().start();
    this.zkTestServer.start();
    // Start Pravega Service.
    this.serviceBuilder = ServiceBuilder.newInMemoryBuilder(ServiceBuilderConfig.getDefaultConfig());
    this.serviceBuilder.initialize();
    StreamSegmentStore store = serviceBuilder.createStreamSegmentService();
    TableStore tableStore = serviceBuilder.createTableStoreService();
    this.server = new PravegaConnectionListener(enableTls, false, "localhost", servicePort, store, tableStore, SegmentStatsRecorder.noOp(), TableSegmentStatsRecorder.noOp(), new PassingTokenVerifier(), pathToConfig() + SecurityConfigDefaults.TLS_SERVER_CERT_FILE_NAME, pathToConfig() + SecurityConfigDefaults.TLS_SERVER_PRIVATE_KEY_FILE_NAME, true, serviceBuilder.getLowPriorityExecutor(), SecurityConfigDefaults.TLS_PROTOCOL_VERSION);
    this.server.startListening();
    log.info("Started Pravega Service");
    this.adminListener = new AdminConnectionListener(enableTls, false, "localhost", adminPort, store, tableStore, new PassingTokenVerifier(), pathToConfig() + SecurityConfigDefaults.TLS_SERVER_CERT_FILE_NAME, pathToConfig() + SecurityConfigDefaults.TLS_SERVER_PRIVATE_KEY_FILE_NAME, SecurityConfigDefaults.TLS_PROTOCOL_VERSION);
    this.adminListener.startListening();
    log.info("AdminConnectionListener started successfully.");
    // Start Controller.
    this.controllerWrapper = new ControllerWrapper(this.zkTestServer.getConnectString(), false, true, controllerRPCPort, "localhost", servicePort, Config.HOST_STORE_CONTAINER_COUNT, controllerRESTPort, enableAuth, pathToConfig() + SecurityConfigDefaults.AUTH_HANDLER_INPUT_FILE_NAME, "secret", true, 600, enableTls, SecurityConfigDefaults.TLS_PROTOCOL_VERSION, pathToConfig() + SecurityConfigDefaults.TLS_SERVER_CERT_FILE_NAME, pathToConfig() + SecurityConfigDefaults.TLS_SERVER_PRIVATE_KEY_FILE_NAME, pathToConfig() + SecurityConfigDefaults.TLS_SERVER_KEYSTORE_NAME, pathToConfig() + SecurityConfigDefaults.TLS_PASSWORD_FILE_NAME);
    this.controllerWrapper.awaitRunning();
    this.controllerWrapper.getController().createScope(scope).get();
    log.info("Initialized Pravega Controller");
}

Example 58 with TableStore

use of io.pravega.segmentstore.contracts.tables.TableStore in project pravega by pravega.

the class EndToEndAutoScaleUpTest method main.

public static void main(String[] args) throws Exception {
    try {
        @Cleanup TestingServer zkTestServer = new TestingServerStarter().start();
        int port = Config.SERVICE_PORT;
        @Cleanup ControllerWrapper controllerWrapper = new ControllerWrapper(zkTestServer.getConnectString(), port, false);
        Controller controller = controllerWrapper.getController();
        ClientFactoryImpl internalCF = new ClientFactoryImpl(NameUtils.INTERNAL_SCOPE_NAME, controller, new SocketConnectionFactoryImpl(ClientConfig.builder().build()));
        @Cleanup("shutdownNow") val executor = ExecutorServiceHelpers.newScheduledThreadPool(1, "test");
        @Cleanup ServiceBuilder serviceBuilder = ServiceBuilder.newInMemoryBuilder(ServiceBuilderConfig.getDefaultConfig());
        serviceBuilder.initialize();
        StreamSegmentStore store = serviceBuilder.createStreamSegmentService();
        TableStore tableStore = serviceBuilder.createTableStoreService();
        @Cleanup AutoScaleMonitor autoScaleMonitor = new AutoScaleMonitor(store, internalCF, AutoScalerConfig.builder().with(AutoScalerConfig.MUTE_IN_SECONDS, 0).with(AutoScalerConfig.COOLDOWN_IN_SECONDS, 0).build());
        @Cleanup PravegaConnectionListener server = new PravegaConnectionListener(false, false, "localhost", 12345, store, tableStore, autoScaleMonitor.getStatsRecorder(), autoScaleMonitor.getTableSegmentStatsRecorder(), null, null, null, true, serviceBuilder.getLowPriorityExecutor(), Config.TLS_PROTOCOL_VERSION.toArray(new String[Config.TLS_PROTOCOL_VERSION.size()]));
        server.startListening();
        controllerWrapper.awaitRunning();
        controllerWrapper.getControllerService().createScope("test", 0L).get();
        controller.createStream("test", "test", CONFIG).get();
        @Cleanup MockClientFactory clientFactory = new MockClientFactory("test", controller, internalCF.getConnectionPool());
        // Mocking pravega service by putting scale up and scale down requests for the stream
        @Cleanup EventStreamWriter<String> test = clientFactory.createEventWriter("test", new JavaSerializer<>(), EventWriterConfig.builder().build());
        // keep writing. Scale should happen
        long start = System.currentTimeMillis();
        char[] chars = new char[1];
        Arrays.fill(chars, 'a');
        String str = new String(chars);
        CompletableFuture.runAsync(() -> {
            while (System.currentTimeMillis() - start < Duration.ofMinutes(3).toMillis()) {
                try {
                    test.writeEvent("0", str).get();
                } catch (Throwable e) {
                    System.err.println("test exception writing events " + e.getMessage());
                    break;
                }
            }
        });
        Retry.withExpBackoff(10, 10, 100, 10000).retryingOn(NotDoneException.class).throwingOn(RuntimeException.class).runAsync(() -> controller.getCurrentSegments("test", "test").thenAccept(streamSegments -> {
            if (streamSegments.getSegments().size() > 3) {
                System.err.println("Success");
                log.info("Success");
                System.exit(0);
            } else {
                throw new NotDoneException();
            }
        }), executor).exceptionally(e -> {
            System.err.println("Failure");
            log.error("Failure");
            System.exit(1);
            return null;
        }).get();
    } catch (Throwable e) {
        System.err.print("Test failed with exception: " + e.getMessage());
        System.exit(-1);
    }
    System.exit(0);
}

Example 59 with TableStore

use of io.pravega.segmentstore.contracts.tables.TableStore in project pravega by pravega.

the class EndToEndAutoScaleDownTest method main.

public static void main(String[] args) throws Exception {
    try {
        @Cleanup TestingServer zkTestServer = new TestingServerStarter().start();
        int port = Config.SERVICE_PORT;
        @Cleanup ControllerWrapper controllerWrapper = new ControllerWrapper(zkTestServer.getConnectString(), port, false);
        Controller controller = controllerWrapper.getController();
        controllerWrapper.getControllerService().createScope(NameUtils.INTERNAL_SCOPE_NAME, 0L).get();
        ClientFactoryImpl internalCF = new ClientFactoryImpl(NameUtils.INTERNAL_SCOPE_NAME, controller, new SocketConnectionFactoryImpl(ClientConfig.builder().build()));
        ServiceBuilder serviceBuilder = ServiceBuilder.newInMemoryBuilder(ServiceBuilderConfig.getDefaultConfig());
        serviceBuilder.initialize();
        StreamSegmentStore store = serviceBuilder.createStreamSegmentService();
        TableStore tableStore = serviceBuilder.createTableStoreService();
        @Cleanup AutoScaleMonitor autoScaleMonitor = new AutoScaleMonitor(store, internalCF, AutoScalerConfig.builder().with(AutoScalerConfig.MUTE_IN_SECONDS, 0).with(AutoScalerConfig.COOLDOWN_IN_SECONDS, 0).with(AutoScalerConfig.CACHE_CLEANUP_IN_SECONDS, 5).with(AutoScalerConfig.CACHE_EXPIRY_IN_SECONDS, 30).build());
        @Cleanup PravegaConnectionListener server = new PravegaConnectionListener(false, false, "localhost", 12345, store, tableStore, autoScaleMonitor.getStatsRecorder(), autoScaleMonitor.getTableSegmentStatsRecorder(), null, null, null, true, serviceBuilder.getLowPriorityExecutor(), Config.TLS_PROTOCOL_VERSION.toArray(new String[Config.TLS_PROTOCOL_VERSION.size()]));
        server.startListening();
        controllerWrapper.awaitRunning();
        controllerWrapper.getControllerService().createScope("test", 0L).get();
        controller.createStream("test", "test", CONFIG).get();
        Stream stream = new StreamImpl("test", "test");
        Map<Double, Double> map = new HashMap<>();
        map.put(0.0, 0.33);
        map.put(0.33, 0.66);
        map.put(0.66, 1.0);
        @Cleanup("shutdownNow") ScheduledExecutorService executor = ExecutorServiceHelpers.newScheduledThreadPool(1, "test");
        controller.scaleStream(stream, Collections.singletonList(0L), map, executor).getFuture().get();
        Retry.withExpBackoff(10, 10, 100, 10000).retryingOn(NotDoneException.class).throwingOn(RuntimeException.class).runAsync(() -> controller.getCurrentSegments("test", "test").thenAccept(streamSegments -> {
            if (streamSegments.getSegments().size() < 3) {
                System.err.println("Success");
                log.info("Success");
                System.exit(0);
            } else {
                throw new NotDoneException();
            }
        }), executor).exceptionally(e -> {
            System.err.println("Failure");
            log.error("Failure");
            System.exit(1);
            return null;
        }).get();
    } catch (Throwable e) {
        System.err.print("Test failed with exception: " + e.getMessage());
        System.exit(-1);
    }
    System.exit(0);
}

Example 60 with TableStore

use of io.pravega.segmentstore.contracts.tables.TableStore in project pravega by pravega.

the class MetricsTest method setup.

@Before
public void setup() throws Exception {
    final int controllerPort = TestUtils.getAvailableListenPort();
    final String serviceHost = "localhost";
    final int servicePort = TestUtils.getAvailableListenPort();
    final int containerCount = 4;
    // 1. Start Metrics service
    log.info("Initializing metrics provider ...");
    MetricsConfig metricsConfig = MetricsConfig.builder().with(MetricsConfig.ENABLE_STATISTICS, true).with(MetricsConfig.ENABLE_STATSD_REPORTER, false).build();
    metricsConfig.setDynamicCacheEvictionDuration(Duration.ofSeconds(2));
    MetricsProvider.initialize(metricsConfig);
    statsProvider = MetricsProvider.getMetricsProvider();
    statsProvider.startWithoutExporting();
    log.info("Metrics Stats provider is started");
    // 2. Start ZK
    this.zkTestServer = new TestingServerStarter().start();
    // 3. Start Pravega SegmentStore service.
    serviceBuilder = ServiceBuilder.newInMemoryBuilder(ServiceBuilderConfig.getDefaultConfig());
    serviceBuilder.initialize();
    StreamSegmentStore store = serviceBuilder.createStreamSegmentService();
    monitor = new AutoScaleMonitor(store, AutoScalerConfig.builder().build());
    TableStore tableStore = serviceBuilder.createTableStoreService();
    this.server = new PravegaConnectionListener(false, false, "localhost", servicePort, store, tableStore, monitor.getStatsRecorder(), monitor.getTableSegmentStatsRecorder(), new PassingTokenVerifier(), null, null, true, this.serviceBuilder.getLowPriorityExecutor(), SecurityConfigDefaults.TLS_PROTOCOL_VERSION);
    this.server.startListening();
    // 4. Start Pravega Controller service
    this.controllerWrapper = new ControllerWrapper(zkTestServer.getConnectString(), false, controllerPort, serviceHost, servicePort, containerCount);
    this.controllerWrapper.awaitRunning();
    this.controller = controllerWrapper.getController();
}