Examples with SegmentMetadata com.linkedin.pinot.common.segment.SegmentMetadata used on opensource projects

Example 21 with SegmentMetadata

use of com.linkedin.pinot.common.segment.SegmentMetadata in project pinot by linkedin.

the class StarTreeQueryGenerator method main.

/**
   * Given star tree segments directory and number of queries, generate star tree queries.
   * Usage: StarTreeQueryGenerator starTreeSegmentsDirectory numQueries
   *
   * @param args arguments.
   */
public static void main(String[] args) throws Exception {
    if (args.length != 2) {
        System.err.println("Usage: StarTreeQueryGenerator starTreeSegmentsDirectory numQueries");
        return;
    }
    // Get segment metadata for the first segment to get table name and verify query is fit for star tree.
    File segmentsDir = new File(args[0]);
    Preconditions.checkState(segmentsDir.exists());
    Preconditions.checkState(segmentsDir.isDirectory());
    File[] segments = segmentsDir.listFiles();
    Preconditions.checkNotNull(segments);
    File segment = segments[0];
    IndexSegment indexSegment = Loaders.IndexSegment.load(segment, ReadMode.heap);
    SegmentMetadata segmentMetadata = indexSegment.getSegmentMetadata();
    String tableName = segmentMetadata.getTableName();
    // Set up star tree query generator.
    int numQueries = Integer.parseInt(args[1]);
    SegmentInfoProvider infoProvider = new SegmentInfoProvider(args[0]);
    StarTreeQueryGenerator generator = new StarTreeQueryGenerator(tableName, infoProvider.getSingleValueDimensionColumns(), infoProvider.getMetricColumns(), infoProvider.getSingleValueDimensionValuesMap());
    Pql2Compiler compiler = new Pql2Compiler();
    for (int i = 0; i < numQueries; i++) {
        String query = generator.nextQuery();
        System.out.println(query);
        // Verify that query is fit for star tree.
        BrokerRequest brokerRequest = compiler.compileToBrokerRequest(query);
        Preconditions.checkState(RequestUtils.isFitForStarTreeIndex(segmentMetadata, RequestUtils.generateFilterQueryTree(brokerRequest), brokerRequest));
    }
}

Example 22 with SegmentMetadata

use of com.linkedin.pinot.common.segment.SegmentMetadata in project pinot by linkedin.

the class PinotSegmentUploadRestletResource method uploadSegment.

@HttpVerb("post")
@Summary("Uploads a segment")
@Tags({ "segment" })
@Paths({ "/segments", "/segments/" })
@Responses({ @Response(statusCode = "200", description = "The segment was successfully uploaded"), @Response(statusCode = "403", description = "Forbidden operation typically because it exceeds configured quota"), @Response(statusCode = "500", description = "There was an error when uploading the segment") })
private Representation uploadSegment(File indexDir, File dataFile, String downloadUrl) throws ConfigurationException, IOException, JSONException {
    final SegmentMetadata metadata = new SegmentMetadataImpl(indexDir);
    final File tableDir = new File(baseDataDir, metadata.getTableName());
    String tableName = metadata.getTableName();
    File segmentFile = new File(tableDir, dataFile.getName());
    OfflineTableConfig offlineTableConfig = (OfflineTableConfig) ZKMetadataProvider.getOfflineTableConfig(_pinotHelixResourceManager.getPropertyStore(), tableName);
    if (offlineTableConfig == null) {
        LOGGER.info("Missing configuration for table: {} in helix", metadata.getTableName());
        setStatus(Status.CLIENT_ERROR_NOT_FOUND);
        StringRepresentation repr = new StringRepresentation("{\"error\" : \"Missing table: " + tableName + "\"}");
        repr.setMediaType(MediaType.APPLICATION_JSON);
        return repr;
    }
    StorageQuotaChecker.QuotaCheckerResponse quotaResponse = checkStorageQuota(indexDir, metadata, offlineTableConfig);
    if (!quotaResponse.isSegmentWithinQuota) {
        // this is not an "error" hence we don't increment segment upload errors
        LOGGER.info("Rejecting segment upload for table: {}, segment: {}, reason: {}", metadata.getTableName(), metadata.getName(), quotaResponse.reason);
        setStatus(Status.CLIENT_ERROR_FORBIDDEN);
        StringRepresentation repr = new StringRepresentation("{\"error\" : \"" + quotaResponse.reason + "\"}");
        repr.setMediaType(MediaType.APPLICATION_JSON);
        return repr;
    }
    PinotResourceManagerResponse response;
    if (!isSegmentTimeValid(metadata)) {
        response = new PinotResourceManagerResponse("Invalid segment start/end time", false);
    } else {
        if (downloadUrl == null) {
            // serve the data downloading from Pinot Servers.
            if (segmentFile.exists()) {
                FileUtils.deleteQuietly(segmentFile);
            }
            FileUtils.moveFile(dataFile, segmentFile);
            downloadUrl = ControllerConf.constructDownloadUrl(tableName, dataFile.getName(), vip);
        }
        // TODO: this will read table configuration again from ZK. We should optimize that
        response = _pinotHelixResourceManager.addSegment(metadata, downloadUrl);
    }
    if (response.isSuccessful()) {
        setStatus(Status.SUCCESS_OK);
    } else {
        ControllerRestApplication.getControllerMetrics().addMeteredGlobalValue(ControllerMeter.CONTROLLER_SEGMENT_UPLOAD_ERROR, 1L);
        setStatus(Status.SERVER_ERROR_INTERNAL);
    }
    return new StringRepresentation(response.toJSON().toString());
}

Example 23 with SegmentMetadata

use of com.linkedin.pinot.common.segment.SegmentMetadata in project pinot by linkedin.

the class AggregationGroupByPlanNode method run.

@Override
public Operator run() {
    TransformExpressionOperator transformOperator = (TransformExpressionOperator) _transformPlanNode.run();
    SegmentMetadata segmentMetadata = _indexSegment.getSegmentMetadata();
    return new AggregationGroupByOperator(AggregationFunctionUtils.getAggregationFunctionContexts(_aggregationInfos, segmentMetadata), _groupBy, _numGroupsLimit, transformOperator, segmentMetadata.getTotalRawDocs());
}

Example 24 with SegmentMetadata

use of com.linkedin.pinot.common.segment.SegmentMetadata in project pinot by linkedin.

the class BrokerRequestPreProcessor method rewriteFastHllColumnName.

/**
   * Rewrite 'fasthll' column name.
   *
   * @param indexSegments list of index segments.
   * @param aggregationsInfo list of aggregation info.
   */
private static void rewriteFastHllColumnName(List<IndexSegment> indexSegments, List<AggregationInfo> aggregationsInfo) {
    // Consistent check.
    for (AggregationInfo aggregationInfo : aggregationsInfo) {
        if (aggregationInfo.getAggregationType().equalsIgnoreCase("fasthll")) {
            String column = aggregationInfo.getAggregationParams().get("column").trim();
            boolean isFirstSegment = true;
            String firstSegmentName = null;
            String hllDerivedColumn = null;
            for (IndexSegment indexSegment : indexSegments) {
                SegmentMetadata segmentMetadata = indexSegment.getSegmentMetadata();
                if (isFirstSegment) {
                    // Use metadata from first index segment to perform rewrite.
                    isFirstSegment = false;
                    firstSegmentName = segmentMetadata.getName();
                    hllDerivedColumn = segmentMetadata.getDerivedColumn(column, MetricFieldSpec.DerivedMetricType.HLL);
                    if (hllDerivedColumn != null) {
                        aggregationInfo.getAggregationParams().put("column", hllDerivedColumn);
                    }
                } else {
                    // Perform consistency check on other index segments.
                    String hllDerivedColumnToCheck = segmentMetadata.getDerivedColumn(column, MetricFieldSpec.DerivedMetricType.HLL);
                    if (!Objects.equals(hllDerivedColumn, hllDerivedColumnToCheck)) {
                        throw new RuntimeException("Found inconsistency HLL derived column name. In segment " + firstSegmentName + ": " + hllDerivedColumn + "; In segment " + segmentMetadata.getName() + ": " + hllDerivedColumnToCheck);
                    }
                }
            }
        }
    }
}

Example 25 with SegmentMetadata

use of com.linkedin.pinot.common.segment.SegmentMetadata in project pinot by linkedin.

the class RetentionManagerTest method testRetentionWithDaysTimeUnit.

/**
   * Test with daysSinceEpoch time unit and make 10 segments with expired time value, 10 segments
   * with the day after tomorrow's time stamp.
   * @throws JSONException
   * @throws UnsupportedEncodingException
   * @throws IOException
   * @throws InterruptedException
   */
@Test
public void testRetentionWithDaysTimeUnit() throws JSONException, UnsupportedEncodingException, IOException, InterruptedException {
    _retentionManager = new RetentionManager(_pinotHelixResourceManager, 5);
    _retentionManager.start();
    long theDayAfterTomorrowSinceEpoch = System.currentTimeMillis() / 1000 / 60 / 60 / 24 + 2;
    for (int i = 0; i < 10; ++i) {
        SegmentMetadata segmentMetadata = getTimeSegmentMetadataImpl("15544", "15544", TimeUnit.DAYS.toString());
        registerSegmentMetadata(segmentMetadata);
        Thread.sleep(100);
    }
    for (int i = 0; i < 10; ++i) {
        SegmentMetadata segmentMetadata = getTimeSegmentMetadataImpl(theDayAfterTomorrowSinceEpoch + "", theDayAfterTomorrowSinceEpoch + "", TimeUnit.DAYS.toString());
        registerSegmentMetadata(segmentMetadata);
        Thread.sleep(100);
    }
    validate(20, _offlineTableName, 10, true);
    cleanupSegments(_offlineTableName);
}