Examples with AbstractTableConfig com.linkedin.pinot.common.config.AbstractTableConfig used on opensource projects

Example 31 with AbstractTableConfig

use of com.linkedin.pinot.common.config.AbstractTableConfig in project pinot by linkedin.

the class PinotTableRestletResource method post.

@Override
@Post("json")
public Representation post(Representation entity) {
    AbstractTableConfig config = null;
    try {
        String jsonRequest = entity.getText();
        config = AbstractTableConfig.init(jsonRequest);
        try {
            addTable(config);
        } catch (Exception e) {
            LOGGER.error("Caught exception while adding table", e);
            ControllerRestApplication.getControllerMetrics().addMeteredGlobalValue(ControllerMeter.CONTROLLER_TABLE_ADD_ERROR, 1L);
            setStatus(Status.SERVER_ERROR_INTERNAL);
            return new StringRepresentation("Failed: " + e.getMessage());
        }
        return new StringRepresentation("Success");
    } catch (Exception e) {
        LOGGER.error("error reading/serializing requestJSON", e);
        setStatus(Status.CLIENT_ERROR_BAD_REQUEST);
        return new StringRepresentation("Failed: " + e.getMessage());
    }
}

Example 32 with AbstractTableConfig

use of com.linkedin.pinot.common.config.AbstractTableConfig in project pinot by linkedin.

the class PinotTableSchema method getTableSchema.

@HttpVerb("get")
@Summary("Gets schema for a table")
@Tags({ "schema", "table" })
@Paths({ "/tables/{tableName}/schema", "/tables/{tableName}/schema/" })
private Representation getTableSchema(@Parameter(name = "tableName", in = "path", description = "Table name for which to get the schema", required = true) String tableName) {
    if (_pinotHelixResourceManager.hasRealtimeTable(tableName)) {
        try {
            AbstractTableConfig config = _pinotHelixResourceManager.getTableConfig(tableName, TableType.REALTIME);
            return new StringRepresentation(_pinotHelixResourceManager.getSchema(config.getValidationConfig().getSchemaName()).getJSONSchema().toString());
        } catch (Exception e) {
            LOGGER.error("Caught exception while fetching schema for a realtime table : {} ", tableName, e);
            ControllerRestApplication.getControllerMetrics().addMeteredGlobalValue(ControllerMeter.CONTROLLER_TABLE_SCHEMA_GET_ERROR, 1L);
            setStatus(Status.SERVER_ERROR_INTERNAL);
            return PinotSegmentUploadRestletResource.exceptionToStringRepresentation(e);
        }
    } else {
        AbstractTableConfig config;
        try {
            config = _pinotHelixResourceManager.getTableConfig(tableName, TableType.OFFLINE);
            String schemaName = config.getValidationConfig().getSchemaName();
            Schema schema = null;
            if (schemaName != null && !schemaName.isEmpty()) {
                schema = _pinotHelixResourceManager.getSchema(schemaName);
            }
            if (schema == null) {
                setStatus(Status.CLIENT_ERROR_NOT_FOUND);
                StringRepresentation repr = new StringRepresentation("{\"error\": \"Schema " + schemaName + " not found\"");
                repr.setMediaType(MediaType.APPLICATION_JSON);
                return repr;
            }
            return new StringRepresentation(schema.getJSONSchema().toString());
        } catch (Exception e) {
            LOGGER.error("Caught exception while fetching schema for a offline table : {} ", tableName, e);
            ControllerRestApplication.getControllerMetrics().addMeteredGlobalValue(ControllerMeter.CONTROLLER_TABLE_SCHEMA_GET_ERROR, 1L);
            setStatus(Status.SERVER_ERROR_INTERNAL);
            return PinotSegmentUploadRestletResource.exceptionToStringRepresentation(e);
        }
    }
}

Example 33 with AbstractTableConfig

use of com.linkedin.pinot.common.config.AbstractTableConfig in project pinot by linkedin.

the class PinotTableSegmentConfigs method updateSegmentConfig.

@Deprecated
@HttpVerb("put")
@Summary("DEPRECATED: Updates the segment configuration (validation and retention) for a table")
@Tags({ "table" })
@Paths({ "/tables/{tableName}/segmentConfigs" })
private Representation updateSegmentConfig(@Parameter(name = "tableName", in = "path", description = "The name of the table for which to update the segment configuration", required = true) String tableName, Representation entity) throws Exception {
    AbstractTableConfig config = AbstractTableConfig.init(entity.getText());
    _pinotHelixResourceManager.updateSegmentsValidationAndRetentionConfigFor(config.getTableName(), TableType.valueOf(config.getTableType().toUpperCase()), config.getValidationConfig());
    return new StringRepresentation("done");
}

Example 34 with AbstractTableConfig

use of com.linkedin.pinot.common.config.AbstractTableConfig in project pinot by linkedin.

the class RoutingTableTest method testTableConfigRoutingTableSelector.

@Test
public void testTableConfigRoutingTableSelector() throws Exception {
    FakePropertyStore fakePropertyStore = new FakePropertyStore();
    TableConfigRoutingTableSelector tableConfigRoutingTableSelector = new TableConfigRoutingTableSelector();
    tableConfigRoutingTableSelector.init(null, fakePropertyStore);
    String propertyStoreEntry = "{\n" + "        \"tableName\":\"fakeTable\",\n" + "        \"segmentsConfig\": {\n" + "            \"retentionTimeUnit\":\"DAYS\",\n" + "            \"retentionTimeValue\":\"5\",\n" + "            \"segmentPushFrequency\":\"daily\",\n" + "            \"segmentPushType\":\"APPEND\",\n" + "            \"replication\": \"2\",\n" + "            \"schemaName\": \"fakeSchema\",\n" + "            \"timeColumnName\": \"time\",\n" + "            \"timeType\": \"MINUTES\",\n" + "            \"segmentAssignmentStrategy\": \"BalanceNumSegmentAssignmentStrategy\"\n" + "         },\n" + "         \"tableIndexConfig\": {\n" + "            \"invertedIndexColumns\": [\"columnA\",\"columnB\"],\n" + "            \"loadMode\": \"MMAP\",\n" + "            \"lazyLoad\": \"false\",\n" + "            \"streamConfigs\": {\n" + "               \"streamType\": \"kafka\",\n" + "               \"stream.kafka.consumer.type\": \"highLevel,simple\",\n" + "               \"stream.kafka.topic.name\": \"FakeTopic\",\n" + "               \"stream.kafka.decoder.class.name\": \"com.linkedin.pinot.core.realtime.impl.kafka.KafkaAvroMessageDecoder\",\n" + "\n" + "               \"stream.kafka.zk.broker.url\": \"fakezk:1234\",\n" + "               \"stream.kafka.hlc.zk.connect.string\": \"fakezk:1234\",\n" + "               \"stream.kafka.decoder.prop.schema.registry.rest.url\": \"fakeschemaregistry:1234\",\n" + "               \"stream.kafka.decoder.prop.schema.registry.schema.name\": \"FakeSchema\"\n" + "             }\n" + "          },\n" + "          \"tenants\": {\n" + "              \"broker\":\"fakebroker\",\n" + "              \"server\":\"fakeserver\"\n" + "          },\n" + "          \"tableType\":\"REALTIME\",\n" + "          \"metadata\": {\n" + "              \"customConfigs\": {\n" + "                  \"routing.llc.percentage\": \"50.0\"\n" + "               }\n" + "          }\n" + "    }\n" + "}";
    AbstractTableConfig tableConfig = AbstractTableConfig.init(propertyStoreEntry);
    fakePropertyStore.setContents("/CONFIGS/TABLE/fakeTable_REALTIME", AbstractTableConfig.toZnRecord(tableConfig));
    tableConfigRoutingTableSelector.registerTable("fakeTable_REALTIME");
    int llcCount = 0;
    for (int i = 0; i < 10000; ++i) {
        if (tableConfigRoutingTableSelector.shouldUseLLCRouting("fakeTable_REALTIME")) {
            llcCount++;
        }
    }
    Assert.assertTrue(4500 <= llcCount && llcCount <= 5500, "Expected approximately 50% probability of picking LLC, got " + llcCount / 100.0 + " %");
    tableConfig.getCustomConfigs().setCustomConfigs(Collections.singletonMap("routing.llc.percentage", "0"));
    fakePropertyStore.setContents("/CONFIGS/TABLE/fakeTable_REALTIME", AbstractTableConfig.toZnRecord(tableConfig));
    llcCount = 0;
    for (int i = 0; i < 10000; ++i) {
        if (tableConfigRoutingTableSelector.shouldUseLLCRouting("fakeTable_REALTIME")) {
            llcCount++;
        }
    }
    Assert.assertEquals(llcCount, 0, "Expected 0% probability of picking LLC, got " + llcCount / 100.0 + " %");
    tableConfig.getCustomConfigs().setCustomConfigs(Collections.singletonMap("routing.llc.percentage", "100"));
    fakePropertyStore.setContents("/CONFIGS/TABLE/fakeTable_REALTIME", AbstractTableConfig.toZnRecord(tableConfig));
    llcCount = 0;
    for (int i = 0; i < 10000; ++i) {
        if (tableConfigRoutingTableSelector.shouldUseLLCRouting("fakeTable_REALTIME")) {
            llcCount++;
        }
    }
    Assert.assertEquals(llcCount, 10000, "Expected 100% probability of picking LLC, got " + llcCount / 100.0 + " %");
}

Example 35 with AbstractTableConfig

use of com.linkedin.pinot.common.config.AbstractTableConfig in project pinot by linkedin.

the class HelixExternalViewBasedTimeBoundaryService method updateTimeBoundaryService.

public synchronized void updateTimeBoundaryService(ExternalView externalView) {
    if (_propertyStore == null) {
        return;
    }
    String tableName = externalView.getResourceName();
    // Do nothing for realtime table.
    if (TableNameBuilder.getTableTypeFromTableName(tableName) == TableType.REALTIME) {
        return;
    }
    Set<String> offlineSegmentsServing = externalView.getPartitionSet();
    if (offlineSegmentsServing.isEmpty()) {
        LOGGER.info("Skipping updating time boundary service for table '{}' with no offline segments.", tableName);
        return;
    }
    AbstractTableConfig offlineTableConfig = ZKMetadataProvider.getOfflineTableConfig(_propertyStore, tableName);
    String timeType = offlineTableConfig.getValidationConfig().getTimeType();
    TimeUnit tableTimeUnit = getTimeUnitFromString(timeType);
    if (tableTimeUnit == null) {
        LOGGER.info("Skipping updating time boundary service for table '{}' with null timeUnit, config time type: {}.", tableName, timeType);
        return;
    }
    // Bulk reading all segment zk-metadata at once is more efficient than reading one at a time.
    List<OfflineSegmentZKMetadata> segmentZKMetadataList = ZKMetadataProvider.getOfflineSegmentZKMetadataListForTable(_propertyStore, tableName);
    long maxTimeValue = computeMaxSegmentEndTimeForTable(segmentZKMetadataList, tableTimeUnit);
    TimeBoundaryInfo timeBoundaryInfo = new TimeBoundaryInfo();
    timeBoundaryInfo.setTimeColumn(offlineTableConfig.getValidationConfig().getTimeColumnName());
    timeBoundaryInfo.setTimeValue(Long.toString(maxTimeValue));
    _timeBoundaryInfoMap.put(tableName, timeBoundaryInfo);
    LOGGER.info("Updated time boundary service for table '{}', maxTime: {}", tableName, maxTimeValue);
}