Examples with OfflineTableConfig com.linkedin.pinot.common.config.OfflineTableConfig used on opensource projects

Example 1 with OfflineTableConfig

use of com.linkedin.pinot.common.config.OfflineTableConfig in project pinot by linkedin.

the class PinotSegmentUploadRestletResource method uploadSegment.

@HttpVerb("post")
@Summary("Uploads a segment")
@Tags({ "segment" })
@Paths({ "/segments", "/segments/" })
@Responses({ @Response(statusCode = "200", description = "The segment was successfully uploaded"), @Response(statusCode = "403", description = "Forbidden operation typically because it exceeds configured quota"), @Response(statusCode = "500", description = "There was an error when uploading the segment") })
private Representation uploadSegment(File indexDir, File dataFile, String downloadUrl) throws ConfigurationException, IOException, JSONException {
    final SegmentMetadata metadata = new SegmentMetadataImpl(indexDir);
    final File tableDir = new File(baseDataDir, metadata.getTableName());
    String tableName = metadata.getTableName();
    File segmentFile = new File(tableDir, dataFile.getName());
    OfflineTableConfig offlineTableConfig = (OfflineTableConfig) ZKMetadataProvider.getOfflineTableConfig(_pinotHelixResourceManager.getPropertyStore(), tableName);
    if (offlineTableConfig == null) {
        LOGGER.info("Missing configuration for table: {} in helix", metadata.getTableName());
        setStatus(Status.CLIENT_ERROR_NOT_FOUND);
        StringRepresentation repr = new StringRepresentation("{\"error\" : \"Missing table: " + tableName + "\"}");
        repr.setMediaType(MediaType.APPLICATION_JSON);
        return repr;
    }
    StorageQuotaChecker.QuotaCheckerResponse quotaResponse = checkStorageQuota(indexDir, metadata, offlineTableConfig);
    if (!quotaResponse.isSegmentWithinQuota) {
        // this is not an "error" hence we don't increment segment upload errors
        LOGGER.info("Rejecting segment upload for table: {}, segment: {}, reason: {}", metadata.getTableName(), metadata.getName(), quotaResponse.reason);
        setStatus(Status.CLIENT_ERROR_FORBIDDEN);
        StringRepresentation repr = new StringRepresentation("{\"error\" : \"" + quotaResponse.reason + "\"}");
        repr.setMediaType(MediaType.APPLICATION_JSON);
        return repr;
    }
    PinotResourceManagerResponse response;
    if (!isSegmentTimeValid(metadata)) {
        response = new PinotResourceManagerResponse("Invalid segment start/end time", false);
    } else {
        if (downloadUrl == null) {
            // serve the data downloading from Pinot Servers.
            if (segmentFile.exists()) {
                FileUtils.deleteQuietly(segmentFile);
            }
            FileUtils.moveFile(dataFile, segmentFile);
            downloadUrl = ControllerConf.constructDownloadUrl(tableName, dataFile.getName(), vip);
        }
        // TODO: this will read table configuration again from ZK. We should optimize that
        response = _pinotHelixResourceManager.addSegment(metadata, downloadUrl);
    }
    if (response.isSuccessful()) {
        setStatus(Status.SUCCESS_OK);
    } else {
        ControllerRestApplication.getControllerMetrics().addMeteredGlobalValue(ControllerMeter.CONTROLLER_SEGMENT_UPLOAD_ERROR, 1L);
        setStatus(Status.SERVER_ERROR_INTERNAL);
    }
    return new StringRepresentation(response.toJSON().toString());
}

Example 2 with OfflineTableConfig

use of com.linkedin.pinot.common.config.OfflineTableConfig in project pinot by linkedin.

the class PinotHelixResourceManager method updateIndexingConfigFor.

public void updateIndexingConfigFor(String tableName, TableType type, IndexingConfig newConfigs) throws Exception {
    String actualTableName = new TableNameBuilder(type).forTable(tableName);
    AbstractTableConfig config;
    if (type == TableType.REALTIME) {
        config = ZKMetadataProvider.getRealtimeTableConfig(getPropertyStore(), actualTableName);
        if (config != null) {
            ((RealtimeTableConfig) config).setIndexConfig(newConfigs);
        }
    } else {
        config = ZKMetadataProvider.getOfflineTableConfig(getPropertyStore(), actualTableName);
        if (config != null) {
            ((OfflineTableConfig) config).setIndexConfig(newConfigs);
        }
    }
    if (config == null) {
        throw new RuntimeException("tableName : " + tableName + " of type : " + type + " not found");
    }
    setTableConfig(config, actualTableName, type);
    if (type == TableType.REALTIME) {
        ensureRealtimeClusterIsSetUp(config, tableName, newConfigs);
    }
}