Examples with QuotaConfig com.linkedin.pinot.common.config.QuotaConfig used on opensource projects

Example 1 with QuotaConfig

use of com.linkedin.pinot.common.config.QuotaConfig in project pinot by linkedin.

the class StorageQuotaChecker method isSegmentStorageWithinQuota.

/**
   * check if the segment represented by segmentFile is within the storage quota
   * @param segmentFile untarred segment. This should not be null.
   *                    segmentFile must exist on disk and must be a directory
   * @param tableNameWithType table name without type (OFFLINE/REALTIME) information
   * @param segmentName name of the segment being added
   * @param timeoutMsec timeout in milliseconds for reading table sizes from server
   *
   */
public QuotaCheckerResponse isSegmentStorageWithinQuota(@Nonnull File segmentFile, @Nonnull String tableNameWithType, @Nonnull String segmentName, @Nonnegative int timeoutMsec) {
    Preconditions.checkNotNull(segmentFile);
    Preconditions.checkNotNull(tableNameWithType);
    Preconditions.checkNotNull(segmentName);
    Preconditions.checkArgument(timeoutMsec > 0, "Timeout value must be > 0, input: %s", timeoutMsec);
    Preconditions.checkArgument(segmentFile.exists(), "Segment file: %s does not exist", segmentFile);
    Preconditions.checkArgument(segmentFile.isDirectory(), "Segment file: %s is not a directory", segmentFile);
    // 1. Read table config
    // 2. read table size from all the servers
    // 3. update predicted segment sizes
    // 4. is the updated size within quota
    QuotaConfig quotaConfig = tableConfig.getQuotaConfig();
    int numReplicas = tableConfig.getValidationConfig().getReplicationNumber();
    final String tableName = tableConfig.getTableName();
    if (quotaConfig == null) {
        // no quota configuration...so ignore for backwards compatibility
        return new QuotaCheckerResponse(true, "Quota configuration not set for table: " + tableNameWithType);
    }
    long allowedStorageBytes = numReplicas * quotaConfig.storageSizeBytes();
    if (allowedStorageBytes < 0) {
        return new QuotaCheckerResponse(true, "Storage quota is not configured for table: " + tableNameWithType);
    }
    long incomingSegmentSizeBytes = FileUtils.sizeOfDirectory(segmentFile);
    // read table size
    TableSizeReader.TableSubTypeSizeDetails tableSubtypeSize = tableSizeReader.getTableSubtypeSize(tableNameWithType, timeoutMsec);
    // If the segment exists(refresh), get the existing size
    TableSizeReader.SegmentSizeDetails sizeDetails = tableSubtypeSize.segments.get(segmentName);
    long existingSegmentSizeBytes = sizeDetails != null ? sizeDetails.estimatedSizeInBytes : 0;
    long estimatedFinalSizeBytes = tableSubtypeSize.estimatedSizeInBytes - existingSegmentSizeBytes + incomingSegmentSizeBytes;
    if (estimatedFinalSizeBytes <= allowedStorageBytes) {
        return new QuotaCheckerResponse(true, String.format("Estimated size: %d bytes is within the configured quota of %d (bytes) for table %s. Incoming segment size: %d (bytes)", estimatedFinalSizeBytes, allowedStorageBytes, tableName, incomingSegmentSizeBytes));
    } else {
        return new QuotaCheckerResponse(false, String.format("Estimated size: %d bytes exceeds the configured quota of %d (bytes) for table %s. Incoming segment size: %d (bytes)", estimatedFinalSizeBytes, allowedStorageBytes, tableName, incomingSegmentSizeBytes));
    }
}

Example 2 with QuotaConfig

use of com.linkedin.pinot.common.config.QuotaConfig in project pinot by linkedin.

the class PinotTableRestletResourceTest method testUpdateTableConfig.

@Test
public void testUpdateTableConfig() throws IOException, JSONException {
    String tableName = "updateTC";
    JSONObject request = ControllerRequestBuilder.buildCreateOfflineTableJSON(tableName, "default", "default", "potato", "DAYS", "DAYS", "5", 2, "BalanceNumSegmentAssignmentStrategy", Collections.<String>emptyList(), "MMAP", "v3");
    ControllerRequestURLBuilder controllerUrlBuilder = ControllerRequestURLBuilder.baseUrl(CONTROLLER_BASE_API_URL);
    sendPostRequest(controllerUrlBuilder.forTableCreate(), request.toString());
    // table creation should succeed
    AbstractTableConfig tableConfig = getTableConfig(tableName, "OFFLINE");
    Assert.assertEquals(tableConfig.getValidationConfig().getRetentionTimeValue(), "5");
    Assert.assertEquals(tableConfig.getValidationConfig().getRetentionTimeUnit(), "DAYS");
    tableConfig.getValidationConfig().setRetentionTimeUnit("HOURS");
    tableConfig.getValidationConfig().setRetentionTimeValue("10");
    String output = sendPutRequest(controllerUrlBuilder.forUpdateTableConfig(tableName), tableConfig.toJSON().toString());
    JSONObject jsonResponse = new JSONObject(output);
    Assert.assertTrue(jsonResponse.has("status"));
    Assert.assertEquals(jsonResponse.getString("status"), "Success");
    AbstractTableConfig modifiedConfig = getTableConfig(tableName, "OFFLINE");
    Assert.assertEquals(modifiedConfig.getValidationConfig().getRetentionTimeUnit(), "HOURS");
    Assert.assertEquals(modifiedConfig.getValidationConfig().getRetentionTimeValue(), "10");
    // Realtime
    JSONObject metadata = new JSONObject();
    metadata.put("streamType", "kafka");
    metadata.put(DataSource.STREAM_PREFIX + "." + Kafka.CONSUMER_TYPE, Kafka.ConsumerType.highLevel.toString());
    metadata.put(DataSource.STREAM_PREFIX + "." + Kafka.TOPIC_NAME, "fakeTopic");
    metadata.put(DataSource.STREAM_PREFIX + "." + Kafka.DECODER_CLASS, "fakeClass");
    metadata.put(DataSource.STREAM_PREFIX + "." + Kafka.ZK_BROKER_URL, "fakeUrl");
    metadata.put(DataSource.STREAM_PREFIX + "." + Kafka.HighLevelConsumer.ZK_CONNECTION_STRING, "potato");
    metadata.put(DataSource.Realtime.REALTIME_SEGMENT_FLUSH_SIZE, Integer.toString(1234));
    metadata.put(DataSource.STREAM_PREFIX + "." + Kafka.KAFKA_CONSUMER_PROPS_PREFIX + "." + Kafka.AUTO_OFFSET_RESET, "smallest");
    request = ControllerRequestBuilder.buildCreateRealtimeTableJSON(tableName, "default", "default", "potato", "DAYS", "DAYS", "5", 2, "BalanceNumSegmentAssignmentStrategy", metadata, "fakeSchema", "fakeColumn", Collections.<String>emptyList(), "MMAP", false);
    sendPostRequest(ControllerRequestURLBuilder.baseUrl(CONTROLLER_BASE_API_URL).forTableCreate(), request.toString());
    tableConfig = getTableConfig(tableName, "REALTIME");
    Assert.assertEquals(tableConfig.getValidationConfig().getRetentionTimeValue(), "5");
    Assert.assertEquals(tableConfig.getValidationConfig().getRetentionTimeUnit(), "DAYS");
    Assert.assertNull(tableConfig.getQuotaConfig());
    QuotaConfig quota = new QuotaConfig();
    quota.setStorage("10G");
    tableConfig.setQuotaConfig(quota);
    sendPutRequest(controllerUrlBuilder.forUpdateTableConfig(tableName), tableConfig.toJSON().toString());
    modifiedConfig = getTableConfig(tableName, "REALTIME");
    Assert.assertEquals(modifiedConfig.getQuotaConfig().getStorage(), "10G");
}