Examples with TableStore io.pravega.segmentstore.contracts.tables.TableStore used on opensource projects

Example 1 with TableStore

use of io.pravega.segmentstore.contracts.tables.TableStore in project pravega by pravega.

the class ServiceStarter method start.

// endregion
// region Service Operation
public void start() throws Exception {
    Exceptions.checkNotClosed(this.closed, this);
    healthServiceManager = new HealthServiceManager(serviceConfig.getHealthCheckInterval());
    healthServiceManager.start();
    log.info("Initializing HealthService ...");
    MetricsConfig metricsConfig = builderConfig.getConfig(MetricsConfig::builder);
    if (metricsConfig.isEnableStatistics()) {
        log.info("Initializing metrics provider ...");
        MetricsProvider.initialize(metricsConfig);
        statsProvider = MetricsProvider.getMetricsProvider();
        statsProvider.start();
    }
    log.info("Initializing ZooKeeper Client ...");
    this.zkClient = createZKClient();
    log.info("Initializing Service Builder ...");
    this.serviceBuilder.initialize();
    log.info("Creating StreamSegmentService ...");
    StreamSegmentStore service = this.serviceBuilder.createStreamSegmentService();
    log.info("Creating TableStoreService ...");
    TableStore tableStoreService = this.serviceBuilder.createTableStoreService();
    log.info("Creating Segment Stats recorder ...");
    autoScaleMonitor = new AutoScaleMonitor(service, builderConfig.getConfig(AutoScalerConfig::builder));
    AutoScalerConfig autoScalerConfig = builderConfig.getConfig(AutoScalerConfig::builder);
    TokenVerifierImpl tokenVerifier = null;
    if (autoScalerConfig.isAuthEnabled()) {
        tokenVerifier = new TokenVerifierImpl(autoScalerConfig.getTokenSigningKey());
    }
    // Log the configuration
    log.info(serviceConfig.toString());
    log.info(autoScalerConfig.toString());
    this.listener = new PravegaConnectionListener(this.serviceConfig.isEnableTls(), this.serviceConfig.isEnableTlsReload(), this.serviceConfig.getListeningIPAddress(), this.serviceConfig.getListeningPort(), service, tableStoreService, autoScaleMonitor.getStatsRecorder(), autoScaleMonitor.getTableSegmentStatsRecorder(), tokenVerifier, this.serviceConfig.getCertFile(), this.serviceConfig.getKeyFile(), this.serviceConfig.isReplyWithStackTraceOnError(), serviceBuilder.getLowPriorityExecutor(), this.serviceConfig.getTlsProtocolVersion(), healthServiceManager);
    this.listener.startListening();
    log.info("PravegaConnectionListener started successfully.");
    if (serviceConfig.isEnableAdminGateway()) {
        this.adminListener = new AdminConnectionListener(this.serviceConfig.isEnableTls(), this.serviceConfig.isEnableTlsReload(), this.serviceConfig.getListeningIPAddress(), this.serviceConfig.getAdminGatewayPort(), service, tableStoreService, tokenVerifier, this.serviceConfig.getCertFile(), this.serviceConfig.getKeyFile(), this.serviceConfig.getTlsProtocolVersion(), healthServiceManager);
        this.adminListener.startListening();
        log.info("AdminConnectionListener started successfully.");
    }
    log.info("StreamSegmentService started.");
    healthServiceManager.register(new ZKHealthContributor(zkClient));
    healthServiceManager.register(new CacheManagerHealthContributor(serviceBuilder.getCacheManager()));
    healthServiceManager.register(new SegmentContainerRegistryHealthContributor(serviceBuilder.getSegmentContainerRegistry()));
    if (this.serviceConfig.isRestServerEnabled()) {
        log.info("Initializing RESTServer ...");
        List<Object> resources = new ArrayList<>();
        resources.add(new HealthImpl(new AuthHandlerManager(serviceConfig.getRestServerConfig()), healthServiceManager.getEndpoint()));
        MetricsProvider.getMetricsProvider().prometheusResource().ifPresent(resources::add);
        restServer = new RESTServer(serviceConfig.getRestServerConfig(), Set.copyOf(resources));
        restServer.startAsync();
        restServer.awaitRunning();
    }
}

Example 2 with TableStore

use of io.pravega.segmentstore.contracts.tables.TableStore in project pravega by pravega.

the class PravegaRequestProcessorTest method testGetTableKeys.

@Test
public void testGetTableKeys() throws Exception {
    // Set up PravegaRequestProcessor instance to execute requests against
    val rnd = new Random(0);
    String tableSegmentName = "testGetTableKeys";
    @Cleanup ServiceBuilder serviceBuilder = newInlineExecutionInMemoryBuilder(getBuilderConfig());
    serviceBuilder.initialize();
    StreamSegmentStore store = serviceBuilder.createStreamSegmentService();
    TableStore tableStore = serviceBuilder.createTableStoreService();
    ServerConnection connection = mock(ServerConnection.class);
    InOrder order = inOrder(connection);
    val recorderMock = mock(TableSegmentStatsRecorder.class);
    PravegaRequestProcessor processor = new PravegaRequestProcessor(store, tableStore, new TrackedConnection(connection), SegmentStatsRecorder.noOp(), recorderMock, new PassingTokenVerifier(), false);
    // Generate keys.
    ArrayList<ArrayView> keys = generateKeys(3, rnd);
    TableEntry e1 = TableEntry.unversioned(keys.get(0), generateValue(rnd));
    TableEntry e2 = TableEntry.unversioned(keys.get(1), generateValue(rnd));
    TableEntry e3 = TableEntry.unversioned(keys.get(2), generateValue(rnd));
    // Create a table segment and add data.
    processor.createTableSegment(new WireCommands.CreateTableSegment(1, tableSegmentName, false, 0, "", 0));
    order.verify(connection).send(new WireCommands.SegmentCreated(1, tableSegmentName));
    verify(recorderMock).createTableSegment(eq(tableSegmentName), any());
    processor.updateTableEntries(new WireCommands.UpdateTableEntries(2, tableSegmentName, "", getTableEntries(asList(e1, e2, e3)), WireCommands.NULL_TABLE_SEGMENT_OFFSET));
    verify(recorderMock).updateEntries(eq(tableSegmentName), eq(3), eq(false), any());
    // 1. Now read the table keys where suggestedKeyCount is equal to number of entries in the Table Store.
    WireCommands.TableIteratorArgs args = new WireCommands.TableIteratorArgs(Unpooled.EMPTY_BUFFER, Unpooled.EMPTY_BUFFER, Unpooled.EMPTY_BUFFER, Unpooled.EMPTY_BUFFER);
    processor.readTableKeys(new WireCommands.ReadTableKeys(3, tableSegmentName, "", 3, args));
    // Capture the WireCommands sent.
    ArgumentCaptor<WireCommand> wireCommandsCaptor = ArgumentCaptor.forClass(WireCommand.class);
    order.verify(connection, times(2)).send(wireCommandsCaptor.capture());
    verify(recorderMock).iterateKeys(eq(tableSegmentName), eq(3), any());
    // Verify the WireCommands.
    List<Long> keyVersions = ((WireCommands.TableEntriesUpdated) wireCommandsCaptor.getAllValues().get(0)).getUpdatedVersions();
    WireCommands.TableKeysRead getTableKeysReadResponse = (WireCommands.TableKeysRead) wireCommandsCaptor.getAllValues().get(1);
    assertTrue(getTableKeysReadResponse.getKeys().stream().map(WireCommands.TableKey::getKeyVersion).collect(Collectors.toList()).containsAll(keyVersions));
    // 2. Now read the table keys where suggestedKeyCount is less than the number of keys in the Table Store.
    processor.readTableKeys(new WireCommands.ReadTableKeys(3, tableSegmentName, "", 1, args));
    // Capture the WireCommands sent.
    ArgumentCaptor<WireCommands.TableKeysRead> tableKeysCaptor = ArgumentCaptor.forClass(WireCommands.TableKeysRead.class);
    order.verify(connection, times(1)).send(tableKeysCaptor.capture());
    verify(recorderMock).iterateKeys(eq(tableSegmentName), eq(1), any());
    // Verify the WireCommands.
    getTableKeysReadResponse = tableKeysCaptor.getAllValues().get(0);
    assertEquals(1, getTableKeysReadResponse.getKeys().size());
    assertTrue(keyVersions.contains(getTableKeysReadResponse.getKeys().get(0).getKeyVersion()));
    // Get the last state.
    ByteBuf state = getTableKeysReadResponse.getContinuationToken();
    args = new WireCommands.TableIteratorArgs(state, Unpooled.EMPTY_BUFFER, Unpooled.EMPTY_BUFFER, Unpooled.EMPTY_BUFFER);
    // 3. Now read the remaining table keys by providing a higher suggestedKeyCount and the state to the iterator.
    processor.readTableKeys(new WireCommands.ReadTableKeys(3, tableSegmentName, "", 3, args));
    // Capture the WireCommands sent.
    tableKeysCaptor = ArgumentCaptor.forClass(WireCommands.TableKeysRead.class);
    order.verify(connection, times(1)).send(tableKeysCaptor.capture());
    verify(recorderMock).iterateKeys(eq(tableSegmentName), eq(1), any());
    // Verify the WireCommands.
    getTableKeysReadResponse = tableKeysCaptor.getAllValues().get(0);
    assertEquals(2, getTableKeysReadResponse.getKeys().size());
    assertTrue(keyVersions.containsAll(getTableKeysReadResponse.getKeys().stream().map(WireCommands.TableKey::getKeyVersion).collect(Collectors.toList())));
}

Example 3 with TableStore

use of io.pravega.segmentstore.contracts.tables.TableStore in project pravega by pravega.

the class PravegaRequestProcessorTest method testCreateTableSegment.

private void testCreateTableSegment(int keyLength) throws Exception {
    // Set up PravegaRequestProcessor instance to execute requests against
    String tableSegmentName = "testCreateTableSegment";
    @Cleanup ServiceBuilder serviceBuilder = newInlineExecutionInMemoryBuilder(getBuilderConfig());
    serviceBuilder.initialize();
    StreamSegmentStore store = serviceBuilder.createStreamSegmentService();
    TableStore tableStore = serviceBuilder.createTableStoreService();
    ServerConnection connection = mock(ServerConnection.class);
    InOrder order = inOrder(connection);
    val recorderMock = mock(TableSegmentStatsRecorder.class);
    PravegaRequestProcessor processor = new PravegaRequestProcessor(store, tableStore, new TrackedConnection(connection), SegmentStatsRecorder.noOp(), recorderMock, new PassingTokenVerifier(), false);
    int requestId = 0;
    // GetInfo with non-existent Table Segment.
    processor.getTableSegmentInfo(new WireCommands.GetTableSegmentInfo(++requestId, tableSegmentName, ""));
    order.verify(connection).send(new WireCommands.NoSuchSegment(requestId, tableSegmentName, "", -1));
    // Execute and Verify createTableSegment calling stack is executed as design.
    processor.createTableSegment(new WireCommands.CreateTableSegment(++requestId, tableSegmentName, false, keyLength, "", 0));
    order.verify(connection).send(new WireCommands.SegmentCreated(requestId, tableSegmentName));
    processor.createTableSegment(new WireCommands.CreateTableSegment(++requestId, tableSegmentName, false, keyLength, "", 0));
    order.verify(connection).send(new WireCommands.SegmentAlreadyExists(requestId, tableSegmentName, ""));
    verify(recorderMock).createTableSegment(eq(tableSegmentName), any());
    verifyNoMoreInteractions(recorderMock);
    // Verify the correct type of Segment is created and that it has the correct roles.
    val si = store.getStreamSegmentInfo(tableSegmentName, PravegaRequestProcessor.TIMEOUT).join();
    val segmentType = SegmentType.fromAttributes(si.getAttributes());
    Assert.assertFalse(segmentType.isInternal() || segmentType.isCritical() || segmentType.isSystem());
    Assert.assertTrue(segmentType.isTableSegment());
    Assert.assertEquals(keyLength > 0, segmentType.isFixedKeyLengthTableSegment());
    val kl = si.getAttributes().getOrDefault(Attributes.ATTRIBUTE_ID_LENGTH, 0L);
    Assert.assertEquals(keyLength, (long) kl);
    // Verify segment info can be retrieved.
    processor.getTableSegmentInfo(new WireCommands.GetTableSegmentInfo(++requestId, tableSegmentName, ""));
    order.verify(connection).send(new WireCommands.TableSegmentInfo(requestId, tableSegmentName, 0, 0, 0, keyLength));
    // Verify invoking GetTableSegmentInfo on a non-existing segment returns NoSuchSegment
    String nonExistingSegment = "nonExistingSegment";
    processor.getTableSegmentInfo(new WireCommands.GetTableSegmentInfo(++requestId, nonExistingSegment, ""));
    order.verify(connection).send(new WireCommands.NoSuchSegment(requestId, nonExistingSegment, "", -1));
    // Verify table segment has correct rollover size
    // Verify default value
    val attributes = si.getAttributes();
    val config = TableExtensionConfig.builder().build();
    Assert.assertEquals((long) attributes.get(Attributes.ROLLOVER_SIZE), config.getDefaultRolloverSize());
    // Verify custom value
    val tableSegmentName1 = "testCreateTableSegmentRolloverSizePositive";
    processor.createTableSegment(new WireCommands.CreateTableSegment(++requestId, tableSegmentName1, false, keyLength, "", 1024 * 1024L));
    val si1 = store.getStreamSegmentInfo(tableSegmentName1, PravegaRequestProcessor.TIMEOUT).join();
    val attributes1 = si1.getAttributes();
    Assert.assertEquals((long) attributes1.get(Attributes.ROLLOVER_SIZE), 1024 * 1024L);
    // Verify invalid value
    val tableSegmentName2 = "testCreateTableSegmentRolloverSizeNegative";
    processor.createTableSegment(new WireCommands.CreateTableSegment(++requestId, tableSegmentName2, false, keyLength, "", -1024L));
    val si2 = store.getStreamSegmentInfo(tableSegmentName2, PravegaRequestProcessor.TIMEOUT).join();
    val attributes2 = si2.getAttributes();
    // fall back to default value
    Assert.assertEquals((long) attributes2.get(Attributes.ROLLOVER_SIZE), config.getDefaultRolloverSize());
}

Example 4 with TableStore

use of io.pravega.segmentstore.contracts.tables.TableStore in project pravega by pravega.

the class PravegaRequestProcessorTest method testDeleteEmptyTable.

@Test(timeout = 30000)
public void testDeleteEmptyTable() throws Exception {
    // Set up PravegaRequestProcessor instance to execute requests against
    String tableSegmentName = "testTable1";
    @Cleanup ServiceBuilder serviceBuilder = newInlineExecutionInMemoryBuilder(getBuilderConfig());
    serviceBuilder.initialize();
    StreamSegmentStore store = serviceBuilder.createStreamSegmentService();
    TableStore tableStore = serviceBuilder.createTableStoreService();
    ServerConnection connection = mock(ServerConnection.class);
    InOrder order = inOrder(connection);
    val recorderMock = mock(TableSegmentStatsRecorder.class);
    PravegaRequestProcessor processor = new PravegaRequestProcessor(store, tableStore, new TrackedConnection(connection), SegmentStatsRecorder.noOp(), recorderMock, new PassingTokenVerifier(), false);
    // Create a table segment.
    processor.createTableSegment(new WireCommands.CreateTableSegment(1, tableSegmentName, false, 0, "", 0));
    order.verify(connection).send(new WireCommands.SegmentCreated(1, tableSegmentName));
    verify(recorderMock).createTableSegment(eq(tableSegmentName), any());
    processor.deleteTableSegment(new WireCommands.DeleteTableSegment(2, tableSegmentName, true, ""));
    order.verify(connection).send(new WireCommands.SegmentDeleted(2, tableSegmentName));
    verify(recorderMock).deleteTableSegment(eq(tableSegmentName), any());
}

Example 5 with TableStore

use of io.pravega.segmentstore.contracts.tables.TableStore in project pravega by pravega.

the class PravegaRequestProcessorTest method testReadTableEntriesDeltaEmpty.

@Test
public void testReadTableEntriesDeltaEmpty() throws Exception {
    // Set up PravegaRequestProcessor instance to execute requests against
    String tableSegmentName = "testReadTableEntriesDelta";
    @Cleanup ServiceBuilder serviceBuilder = newInlineExecutionInMemoryBuilder(getBuilderConfig());
    serviceBuilder.initialize();
    StreamSegmentStore store = serviceBuilder.createStreamSegmentService();
    TableStore tableStore = serviceBuilder.createTableStoreService();
    ServerConnection connection = mock(ServerConnection.class);
    InOrder order = inOrder(connection);
    val recorderMock = mock(TableSegmentStatsRecorder.class);
    PravegaRequestProcessor processor = new PravegaRequestProcessor(store, tableStore, new TrackedConnection(connection), SegmentStatsRecorder.noOp(), recorderMock, new PassingTokenVerifier(), false);
    processor.createTableSegment(new WireCommands.CreateTableSegment(1, tableSegmentName, false, 0, "", 0));
    order.verify(connection).send(new WireCommands.SegmentCreated(1, tableSegmentName));
    verify(recorderMock).createTableSegment(eq(tableSegmentName), any());
    processor.readTableEntriesDelta(new WireCommands.ReadTableEntriesDelta(1, tableSegmentName, "", 0, 3));
    ArgumentCaptor<WireCommand> wireCommandsCaptor = ArgumentCaptor.forClass(WireCommand.class);
    verify(recorderMock).iterateEntries(eq(tableSegmentName), eq(0), any());
}