Examples with GroupBy com.linkedin.pinot.common.request.GroupBy used on opensource projects

Example 1 with GroupBy

use of com.linkedin.pinot.common.request.GroupBy in project pinot by linkedin.

the class RequestUtils method isFitForStarTreeIndex.

/**
   * Returns true for the following, false otherwise:
   * - BrokerRequest debug options have not explicitly disabled use of star tree
   * - Query is not aggregation/group-by
   * - Segment does not contain star tree
   * - The only aggregation function in the query should be in {@link #ALLOWED_AGGREGATION_FUNCTIONS}
   * - All group by columns and predicate columns are materialized
   * - Predicates do not contain any metric columns
   * - Query consists only of simple predicates, conjoined by AND.
   *   <p>
   *   e.g. WHERE d1 = d1v1 AND d2 = d2v2 AND d3 = d3v3 AND d4 between t1,t2
   *   </p>
   *
   */
public static boolean isFitForStarTreeIndex(SegmentMetadata segmentMetadata, FilterQueryTree filterTree, BrokerRequest brokerRequest) {
    // If broker request disables use of star tree, return false.
    if (!isStarTreeEnabledInBrokerRequest(brokerRequest)) {
        return false;
    }
    // Apply the checks in order of their runtime.
    List<AggregationInfo> aggregationsInfo = brokerRequest.getAggregationsInfo();
    // There should have some aggregation
    if (aggregationsInfo == null || aggregationsInfo.isEmpty()) {
        return false;
    }
    // Segment metadata should contain star tree metadata.
    StarTreeMetadata starTreeMetadata = segmentMetadata.getStarTreeMetadata();
    if (starTreeMetadata == null) {
        return false;
    }
    Set<String> metricColumnSet = new HashSet<>(segmentMetadata.getSchema().getMetricNames());
    List<String> skipMaterializationList = starTreeMetadata.getSkipMaterializationForDimensions();
    Set<String> skipMaterializationSet = null;
    if (skipMaterializationList != null && !skipMaterializationList.isEmpty()) {
        skipMaterializationSet = new HashSet<>(skipMaterializationList);
    }
    // Ensure that none of the group-by columns are metric or skipped for materialization.
    GroupBy groupBy = brokerRequest.getGroupBy();
    if (groupBy != null) {
        List<String> groupByColumns = groupBy.getColumns();
        for (String groupByColumn : groupByColumns) {
            if (metricColumnSet.contains(groupByColumn)) {
                return false;
            }
            if (skipMaterializationSet != null && skipMaterializationSet.contains(groupByColumn)) {
                return false;
            }
        }
    }
    // We currently support only limited aggregations
    for (AggregationInfo aggregationInfo : aggregationsInfo) {
        String aggregationFunctionName = aggregationInfo.getAggregationType().toLowerCase();
        if (!ALLOWED_AGGREGATION_FUNCTIONS.contains(aggregationFunctionName)) {
            return false;
        }
    }
    //no metric columns appear in the predicates.
    if (filterTree != null && filterTree.getChildren() != null && !filterTree.getChildren().isEmpty()) {
        //ensure that its AND
        if (filterTree.getOperator() != FilterOperator.AND) {
            return false;
        }
        //ensure that children are not nested further and only one predicate per column
        Set<String> predicateColumns = new HashSet<>();
        for (FilterQueryTree child : filterTree.getChildren()) {
            if (child.getChildren() != null && !child.getChildren().isEmpty()) {
                //star tree index cannot support nested filter predicates
                return false;
            }
            //only one predicate per column is supported
            String column = child.getColumn();
            if (predicateColumns.contains(column)) {
                return false;
            }
            // predicate columns should be materialized.
            if ((skipMaterializationSet != null) && skipMaterializationSet.contains(column)) {
                return false;
            }
            // predicate should not contain metric columns.
            if (metricColumnSet.contains(column)) {
                return false;
            }
            predicateColumns.add(column);
        }
    } else if (filterTree != null) {
        // Predicate column of root node should be materialized.
        String rootColumn = filterTree.getColumn();
        if (skipMaterializationSet != null && skipMaterializationSet.contains(rootColumn)) {
            return false;
        }
        // predicate should not contain metric columns.
        if (metricColumnSet.contains(rootColumn)) {
            return false;
        }
    }
    return true;
}

Example 2 with GroupBy

use of com.linkedin.pinot.common.request.GroupBy in project pinot by linkedin.

the class BrokerRequestSerializationTest method testSerialization.

@Test
public static void testSerialization() throws TException {
    BrokerRequest req = new BrokerRequest();
    // Populate Query Type
    QueryType type = new QueryType();
    type.setHasAggregation(true);
    type.setHasFilter(true);
    type.setHasSelection(true);
    type.setHasGroup_by(true);
    req.setQueryType(type);
    // Populate Query source
    QuerySource s = new QuerySource();
    s.setTableName("dummy");
    req.setQuerySource(s);
    req.setDuration("dummy");
    req.setTimeInterval("dummy");
    //Populate Group-By
    GroupBy groupBy = new GroupBy();
    List<String> columns = new ArrayList<String>();
    columns.add("dummy1");
    columns.add("dummy2");
    groupBy.setColumns(columns);
    groupBy.setTopN(100);
    req.setGroupBy(groupBy);
    //Populate Selections
    Selection sel = new Selection();
    sel.setSize(1);
    SelectionSort s2 = new SelectionSort();
    s2.setColumn("dummy1");
    s2.setIsAsc(true);
    sel.addToSelectionSortSequence(s2);
    sel.addToSelectionColumns("dummy1");
    req.setSelections(sel);
    //Populate FilterQuery
    FilterQuery q1 = new FilterQuery();
    q1.setId(1);
    q1.setColumn("dummy1");
    q1.addToValue("dummy1");
    q1.addToNestedFilterQueryIds(2);
    q1.setOperator(FilterOperator.AND);
    FilterQuery q2 = new FilterQuery();
    q2.setId(2);
    q2.setColumn("dummy2");
    q2.addToValue("dummy2");
    q2.setOperator(FilterOperator.AND);
    FilterQueryMap map = new FilterQueryMap();
    map.putToFilterQueryMap(1, q1);
    map.putToFilterQueryMap(2, q2);
    req.setFilterQuery(q1);
    req.setFilterSubQueryMap(map);
    //Populate Aggregations
    AggregationInfo agg = new AggregationInfo();
    agg.setAggregationType("dummy1");
    agg.putToAggregationParams("key1", "dummy1");
    req.addToAggregationsInfo(agg);
    TSerializer normalSerializer = new TSerializer();
    TSerializer compactSerializer = new TSerializer(new TCompactProtocol.Factory());
    normalSerializer.serialize(req);
    compactSerializer.serialize(req);
//    int numRequests = 100000;
//    TimerContext t = MetricsHelper.startTimer();
//    TSerializer serializer = new TSerializer(new TCompactProtocol.Factory());
//    //TSerializer serializer = new TSerializer();
//    //Compact : Size 183 , Serialization Latency : 0.03361ms
//    // Normal : Size 385 , Serialization Latency : 0.01144ms
//
//    for (int i = 0; i < numRequests; i++) {
//      try {
//        serializer.serialize(req);
//        //System.out.println(s3.length);
//        //break;
//      } catch (TException e) {
//        e.printStackTrace();
//      }
//    }
//    t.stop();
//    System.out.println("Latency is :" + (t.getLatencyMs() / (float) numRequests));
}

Example 3 with GroupBy

use of com.linkedin.pinot.common.request.GroupBy in project pinot by linkedin.

the class StarTreeIndexOperator method initPredicatesToEvaluate.

private void initPredicatesToEvaluate() {
    FilterQueryTree filterTree = RequestUtils.generateFilterQueryTree(brokerRequest);
    // Find all filter columns
    if (filterTree != null) {
        if (filterTree.getChildren() != null && !filterTree.getChildren().isEmpty()) {
            for (FilterQueryTree childFilter : filterTree.getChildren()) {
                // Nested filters are not supported
                assert childFilter.getChildren() == null || childFilter.getChildren().isEmpty();
                processFilterTree(childFilter);
            }
        } else {
            processFilterTree(filterTree);
        }
    }
    // Group by columns, we cannot lose group by columns during traversal
    GroupBy groupBy = brokerRequest.getGroupBy();
    if (groupBy != null) {
        groupByColumns.addAll(groupBy.getColumns());
    }
}

Example 4 with GroupBy

use of com.linkedin.pinot.common.request.GroupBy in project pinot by linkedin.

the class TransformPlanNode method extractColumnsAndTransforms.

/**
   * Helper method to extract projection columns and transform expressions from the given
   * BrokerRequest.
   *  @param brokerRequest BrokerRequest to process
   * @param projectionColumns Output projection columns from broker request
   * @param transformExpressions Output transform expression from broker request
   */
private void extractColumnsAndTransforms(BrokerRequest brokerRequest, Set<String> projectionColumns, Set<String> transformExpressions) {
    if (brokerRequest.isSetAggregationsInfo()) {
        // TODO: Add transform support.
        for (AggregationInfo aggregationInfo : brokerRequest.getAggregationsInfo()) {
            if (!aggregationInfo.getAggregationType().equalsIgnoreCase(AggregationFunctionFactory.AggregationFunctionType.COUNT.getName())) {
                String columns = aggregationInfo.getAggregationParams().get("column").trim();
                projectionColumns.addAll(Arrays.asList(columns.split(",")));
            }
        }
        // Collect all group by related columns.
        if (brokerRequest.isSetGroupBy()) {
            GroupBy groupBy = brokerRequest.getGroupBy();
            List<String> groupByColumns = groupBy.getColumns();
            // GroupByColumns can be null if all group-bys are transforms.
            if (groupByColumns != null) {
                projectionColumns.addAll(groupByColumns);
            }
            // Check null for backward compatibility.
            List<String> expressions = groupBy.getExpressions();
            if (expressions != null) {
                transformExpressions.addAll(expressions);
                if (groupByColumns != null && !groupByColumns.isEmpty()) {
                    transformExpressions.removeAll(groupByColumns);
                }
            }
        }
    } else {
        throw new UnsupportedOperationException("Transforms not supported in selection queries.");
    // TODO: Add transform support.
    // projectionColumns.addAll(brokerRequest.getSelections().getSelectionColumns());
    }
}

Example 5 with GroupBy

use of com.linkedin.pinot.common.request.GroupBy in project pinot by linkedin.

the class SegmentQueryProcessor method pruneSegment.

private boolean pruneSegment(BrokerRequest brokerRequest) {
    // Check if segment belongs to the table being queried.
    if (!_tableName.equals(brokerRequest.getQuerySource().getTableName())) {
        LOGGER.debug("Skipping segment {} from different table {}", _segmentName, _tableName);
        return true;
    }
    // Check if any column in the query does not exist in the segment.
    Set<String> allColumns = _metadata.getAllColumns();
    if (brokerRequest.isSetAggregationsInfo()) {
        for (AggregationInfo aggregationInfo : brokerRequest.getAggregationsInfo()) {
            Map<String, String> aggregationParams = aggregationInfo.getAggregationParams();
            for (String column : aggregationParams.values()) {
                if (column != null && !column.isEmpty() && !column.equals("*") && !allColumns.contains(column)) {
                    LOGGER.debug("Skipping segment '{}', as it does not have column '{}'", _metadata.getName(), column);
                    return true;
                }
            }
            GroupBy groupBy = brokerRequest.getGroupBy();
            if (groupBy != null) {
                for (String column : groupBy.getColumns()) {
                    if (!allColumns.contains(column)) {
                        LOGGER.debug("Skipping segment '{}', as it does not have column '{}'", _metadata.getName(), column);
                        return true;
                    }
                }
            }
        }
    } else {
        if (brokerRequest.isSetSelections()) {
            for (String column : brokerRequest.getSelections().getSelectionColumns()) {
                if (!allColumns.contains(column)) {
                    LOGGER.debug("Skipping segment '{}', as it does not have column '{}'", _metadata.getName(), column);
                    return true;
                }
            }
        }
    }
    return false;
}