Examples with StarTreeMetadata com.linkedin.pinot.common.segment.StarTreeMetadata used on opensource projects

Example 1 with StarTreeMetadata

use of com.linkedin.pinot.common.segment.StarTreeMetadata in project pinot by linkedin.

the class RequestUtils method isFitForStarTreeIndex.

/**
   * Returns true for the following, false otherwise:
   * - BrokerRequest debug options have not explicitly disabled use of star tree
   * - Query is not aggregation/group-by
   * - Segment does not contain star tree
   * - The only aggregation function in the query should be in {@link #ALLOWED_AGGREGATION_FUNCTIONS}
   * - All group by columns and predicate columns are materialized
   * - Predicates do not contain any metric columns
   * - Query consists only of simple predicates, conjoined by AND.
   *   <p>
   *   e.g. WHERE d1 = d1v1 AND d2 = d2v2 AND d3 = d3v3 AND d4 between t1,t2
   *   </p>
   *
   */
public static boolean isFitForStarTreeIndex(SegmentMetadata segmentMetadata, FilterQueryTree filterTree, BrokerRequest brokerRequest) {
    // If broker request disables use of star tree, return false.
    if (!isStarTreeEnabledInBrokerRequest(brokerRequest)) {
        return false;
    }
    // Apply the checks in order of their runtime.
    List<AggregationInfo> aggregationsInfo = brokerRequest.getAggregationsInfo();
    // There should have some aggregation
    if (aggregationsInfo == null || aggregationsInfo.isEmpty()) {
        return false;
    }
    // Segment metadata should contain star tree metadata.
    StarTreeMetadata starTreeMetadata = segmentMetadata.getStarTreeMetadata();
    if (starTreeMetadata == null) {
        return false;
    }
    Set<String> metricColumnSet = new HashSet<>(segmentMetadata.getSchema().getMetricNames());
    List<String> skipMaterializationList = starTreeMetadata.getSkipMaterializationForDimensions();
    Set<String> skipMaterializationSet = null;
    if (skipMaterializationList != null && !skipMaterializationList.isEmpty()) {
        skipMaterializationSet = new HashSet<>(skipMaterializationList);
    }
    // Ensure that none of the group-by columns are metric or skipped for materialization.
    GroupBy groupBy = brokerRequest.getGroupBy();
    if (groupBy != null) {
        List<String> groupByColumns = groupBy.getColumns();
        for (String groupByColumn : groupByColumns) {
            if (metricColumnSet.contains(groupByColumn)) {
                return false;
            }
            if (skipMaterializationSet != null && skipMaterializationSet.contains(groupByColumn)) {
                return false;
            }
        }
    }
    // We currently support only limited aggregations
    for (AggregationInfo aggregationInfo : aggregationsInfo) {
        String aggregationFunctionName = aggregationInfo.getAggregationType().toLowerCase();
        if (!ALLOWED_AGGREGATION_FUNCTIONS.contains(aggregationFunctionName)) {
            return false;
        }
    }
    //no metric columns appear in the predicates.
    if (filterTree != null && filterTree.getChildren() != null && !filterTree.getChildren().isEmpty()) {
        //ensure that its AND
        if (filterTree.getOperator() != FilterOperator.AND) {
            return false;
        }
        //ensure that children are not nested further and only one predicate per column
        Set<String> predicateColumns = new HashSet<>();
        for (FilterQueryTree child : filterTree.getChildren()) {
            if (child.getChildren() != null && !child.getChildren().isEmpty()) {
                //star tree index cannot support nested filter predicates
                return false;
            }
            //only one predicate per column is supported
            String column = child.getColumn();
            if (predicateColumns.contains(column)) {
                return false;
            }
            // predicate columns should be materialized.
            if ((skipMaterializationSet != null) && skipMaterializationSet.contains(column)) {
                return false;
            }
            // predicate should not contain metric columns.
            if (metricColumnSet.contains(column)) {
                return false;
            }
            predicateColumns.add(column);
        }
    } else if (filterTree != null) {
        // Predicate column of root node should be materialized.
        String rootColumn = filterTree.getColumn();
        if (skipMaterializationSet != null && skipMaterializationSet.contains(rootColumn)) {
            return false;
        }
        // predicate should not contain metric columns.
        if (metricColumnSet.contains(rootColumn)) {
            return false;
        }
    }
    return true;
}

Example 2 with StarTreeMetadata

use of com.linkedin.pinot.common.segment.StarTreeMetadata in project pinot by linkedin.

the class TestStarTreeMetadata method testStarTreeMetadata.

/**
   * Read the StarTree metadata and assert that the actual values in the metadata are as expected.
   *
   * @throws Exception
   */
@Test
public void testStarTreeMetadata() throws Exception {
    String segment = INDEX_DIR_NAME + File.separator + SEGMENT_NAME;
    SegmentMetadataImpl segmentMetadata = new SegmentMetadataImpl(new File(segment));
    StarTreeMetadata starTreeMetadata = segmentMetadata.getStarTreeMetadata();
    Assert.assertEquals(starTreeMetadata.getDimensionsSplitOrder(), DIMENSIONS_SPLIT_ORDER);
    Assert.assertEquals(starTreeMetadata.getMaxLeafRecords(), MAX_LEAF_RECORDS);
    Assert.assertEquals(starTreeMetadata.getSkipStarNodeCreationForDimensions(), SKIP_STAR_NODE_CREATION_DIMENSTIONS);
    Assert.assertEquals(starTreeMetadata.getSkipMaterializationCardinality(), SKIP_CARDINALITY_THRESHOLD);
    Assert.assertEquals(starTreeMetadata.getSkipMaterializationForDimensions(), SKIP_MATERIALIZATION_DIMENSIONS);
}

Example 3 with StarTreeMetadata

use of com.linkedin.pinot.common.segment.StarTreeMetadata in project pinot by linkedin.

the class ColumnMetadataTest method testHllIndexRelatedMetadata.

@Test
public void testHllIndexRelatedMetadata() throws Exception {
    SegmentWithHllIndexCreateHelper helper = null;
    try {
        // Build the Segment metadata.
        helper = new SegmentWithHllIndexCreateHelper("testHllIndexRelatedMetadata", getClass().getClassLoader().getResource("data/test_data-sv.avro"), "daysSinceEpoch", TimeUnit.DAYS, "starTreeSegment");
        helper.build(true, new HllConfig(9, new HashSet<String>(Arrays.asList("column7")), "_hllSuffix"));
        // Load segment metadata.
        IndexSegment segment = Loaders.IndexSegment.load(helper.getSegmentDirectory(), ReadMode.mmap);
        SegmentMetadataImpl metadata = (SegmentMetadataImpl) segment.getSegmentMetadata();
        Assert.assertEquals(metadata.getHllLog2m(), 9);
        // Verify Hll Related Info
        StarTreeMetadata starTreeMetadata = metadata.getStarTreeMetadata();
        Assert.assertNotNull(starTreeMetadata);
        ColumnMetadata column = metadata.getColumnMetadataFor("column7_hllSuffix");
        Assert.assertEquals(column.getDerivedMetricType(), MetricFieldSpec.DerivedMetricType.HLL);
        Assert.assertEquals(column.getOriginColumnName(), "column7");
    } finally {
        if (helper != null) {
            helper.cleanTempDir();
        }
    }
}

Example 4 with StarTreeMetadata

use of com.linkedin.pinot.common.segment.StarTreeMetadata in project pinot by linkedin.

the class SegmentMetadataImpl method initStarTreeMetadata.

/**
   * Reads and initializes the star tree metadata from segment metadata properties.
   */
private void initStarTreeMetadata() {
    _starTreeMetadata = new StarTreeMetadata();
    // Set the maxLeafRecords
    String maxLeafRecordsString = _segmentMetadataPropertiesConfiguration.getString(MetadataKeys.StarTree.STAR_TREE_MAX_LEAF_RECORDS);
    if (maxLeafRecordsString != null) {
        _starTreeMetadata.setMaxLeafRecords(Long.valueOf(maxLeafRecordsString));
    }
    // Set the splitOrder
    Iterator<String> iterator = _segmentMetadataPropertiesConfiguration.getList(MetadataKeys.StarTree.STAR_TREE_SPLIT_ORDER).iterator();
    List<String> splitOrder = new ArrayList<String>();
    while (iterator.hasNext()) {
        final String splitColumn = iterator.next();
        splitOrder.add(splitColumn);
    }
    _starTreeMetadata.setDimensionsSplitOrder(splitOrder);
    // Set dimensions for which star node creation is to be skipped.
    iterator = _segmentMetadataPropertiesConfiguration.getList(MetadataKeys.StarTree.STAR_TREE_SKIP_STAR_NODE_CREATION_FOR_DIMENSIONS).iterator();
    List<String> skipStarNodeCreationForDimensions = new ArrayList<String>();
    while (iterator.hasNext()) {
        final String column = iterator.next();
        skipStarNodeCreationForDimensions.add(column);
    }
    _starTreeMetadata.setSkipStarNodeCreationForDimensions(skipStarNodeCreationForDimensions);
    // Set dimensions for which to skip materialization.
    iterator = _segmentMetadataPropertiesConfiguration.getList(MetadataKeys.StarTree.STAR_TREE_SKIP_MATERIALIZATION_FOR_DIMENSIONS).iterator();
    List<String> skipMaterializationForDimensions = new ArrayList<String>();
    while (iterator.hasNext()) {
        final String column = iterator.next();
        skipMaterializationForDimensions.add(column);
    }
    _starTreeMetadata.setSkipMaterializationForDimensions(skipMaterializationForDimensions);
    // Skip skip materialization cardinality.
    String skipMaterializationCardinalityString = _segmentMetadataPropertiesConfiguration.getString(MetadataKeys.StarTree.STAR_TREE_SKIP_MATERIALIZATION_CARDINALITY);
    if (skipMaterializationCardinalityString != null) {
        _starTreeMetadata.setSkipMaterializationCardinality(Long.valueOf(skipMaterializationCardinalityString));
    }
}