Examples with ExecutionStatistics com.linkedin.pinot.core.operator.ExecutionStatistics used on opensource projects

Example 11 with ExecutionStatistics

use of com.linkedin.pinot.core.operator.ExecutionStatistics in project pinot by linkedin.

the class InnerSegmentSelectionSingleValueQueriesTest method testSelectionOnly.

@Test
public void testSelectionOnly() {
    String query = "SELECT" + SELECTION + " FROM testTable";
    // Test query without filter.
    MSelectionOnlyOperator selectionOnlyOperator = getOperatorForQuery(query);
    IntermediateResultsBlock resultsBlock = (IntermediateResultsBlock) selectionOnlyOperator.nextBlock();
    ExecutionStatistics executionStatistics = selectionOnlyOperator.getExecutionStatistics();
    Assert.assertEquals(executionStatistics.getNumDocsScanned(), 10L);
    Assert.assertEquals(executionStatistics.getNumEntriesScannedInFilter(), 0L);
    Assert.assertEquals(executionStatistics.getNumEntriesScannedPostFilter(), 30L);
    Assert.assertEquals(executionStatistics.getNumTotalRawDocs(), 30000L);
    DataSchema selectionDataSchema = resultsBlock.getSelectionDataSchema();
    Assert.assertEquals(selectionDataSchema.size(), 3);
    Assert.assertEquals(selectionDataSchema.getColumnName(0), "column1");
    Assert.assertEquals(selectionDataSchema.getColumnName(2), "column11");
    Assert.assertEquals(selectionDataSchema.getColumnType(0), FieldSpec.DataType.INT);
    Assert.assertEquals(selectionDataSchema.getColumnType(1), FieldSpec.DataType.STRING);
    List<Serializable[]> selectionResult = (List<Serializable[]>) resultsBlock.getSelectionResult();
    Assert.assertEquals(selectionResult.size(), 10);
    Serializable[] firstRow = selectionResult.get(0);
    Assert.assertEquals(firstRow.length, 3);
    Assert.assertEquals(((Integer) firstRow[0]).intValue(), 1578964907);
    Assert.assertEquals((String) firstRow[2], "P");
    // Test query with filter.
    selectionOnlyOperator = getOperatorForQueryWithFilter(query);
    resultsBlock = (IntermediateResultsBlock) selectionOnlyOperator.nextBlock();
    executionStatistics = selectionOnlyOperator.getExecutionStatistics();
    Assert.assertEquals(executionStatistics.getNumDocsScanned(), 10L);
    Assert.assertEquals(executionStatistics.getNumEntriesScannedInFilter(), 48241L);
    Assert.assertEquals(executionStatistics.getNumEntriesScannedPostFilter(), 30L);
    Assert.assertEquals(executionStatistics.getNumTotalRawDocs(), 30000L);
    selectionDataSchema = resultsBlock.getSelectionDataSchema();
    Assert.assertEquals(selectionDataSchema.size(), 3);
    Assert.assertEquals(selectionDataSchema.getColumnName(0), "column1");
    Assert.assertEquals(selectionDataSchema.getColumnName(2), "column11");
    Assert.assertEquals(selectionDataSchema.getColumnType(0), FieldSpec.DataType.INT);
    Assert.assertEquals(selectionDataSchema.getColumnType(1), FieldSpec.DataType.STRING);
    selectionResult = (List<Serializable[]>) resultsBlock.getSelectionResult();
    Assert.assertEquals(selectionResult.size(), 10);
    firstRow = selectionResult.get(0);
    Assert.assertEquals(firstRow.length, 3);
    Assert.assertEquals(((Integer) firstRow[0]).intValue(), 351823652);
    Assert.assertEquals((String) firstRow[2], "t");
}

Example 12 with ExecutionStatistics

use of com.linkedin.pinot.core.operator.ExecutionStatistics in project pinot by linkedin.

the class InnerSegmentSelectionSingleValueQueriesTest method testSelectionOrderBy.

@Test
public void testSelectionOrderBy() {
    String query = "SELECT" + SELECTION + " FROM testTable" + ORDER_BY;
    // Test query without filter.
    MSelectionOrderByOperator selectionOrderByOperator = getOperatorForQuery(query);
    IntermediateResultsBlock resultsBlock = (IntermediateResultsBlock) selectionOrderByOperator.nextBlock();
    ExecutionStatistics executionStatistics = selectionOrderByOperator.getExecutionStatistics();
    Assert.assertEquals(executionStatistics.getNumDocsScanned(), 30000L);
    Assert.assertEquals(executionStatistics.getNumEntriesScannedInFilter(), 0L);
    Assert.assertEquals(executionStatistics.getNumEntriesScannedPostFilter(), 120000L);
    Assert.assertEquals(executionStatistics.getNumTotalRawDocs(), 30000L);
    DataSchema selectionDataSchema = resultsBlock.getSelectionDataSchema();
    Assert.assertEquals(selectionDataSchema.size(), 4);
    Assert.assertEquals(selectionDataSchema.getColumnName(0), "column6");
    Assert.assertEquals(selectionDataSchema.getColumnName(1), "column1");
    Assert.assertEquals(selectionDataSchema.getColumnType(0), FieldSpec.DataType.INT);
    Assert.assertEquals(selectionDataSchema.getColumnType(1), FieldSpec.DataType.INT);
    Queue<Serializable[]> selectionResult = (Queue<Serializable[]>) resultsBlock.getSelectionResult();
    Assert.assertEquals(selectionResult.size(), 10);
    Serializable[] lastRow = selectionResult.peek();
    Assert.assertEquals(lastRow.length, 4);
    Assert.assertEquals(((Integer) lastRow[0]).intValue(), 6043515);
    Assert.assertEquals(((Integer) lastRow[1]).intValue(), 10542595);
    // Test query with filter.
    selectionOrderByOperator = getOperatorForQueryWithFilter(query);
    resultsBlock = (IntermediateResultsBlock) selectionOrderByOperator.nextBlock();
    executionStatistics = selectionOrderByOperator.getExecutionStatistics();
    Assert.assertEquals(executionStatistics.getNumDocsScanned(), 6129L);
    Assert.assertEquals(executionStatistics.getNumEntriesScannedInFilter(), 84134L);
    Assert.assertEquals(executionStatistics.getNumEntriesScannedPostFilter(), 24516L);
    Assert.assertEquals(executionStatistics.getNumTotalRawDocs(), 30000L);
    selectionDataSchema = resultsBlock.getSelectionDataSchema();
    Assert.assertEquals(selectionDataSchema.size(), 4);
    Assert.assertEquals(selectionDataSchema.getColumnName(0), "column6");
    Assert.assertEquals(selectionDataSchema.getColumnName(1), "column1");
    Assert.assertEquals(selectionDataSchema.getColumnType(0), FieldSpec.DataType.INT);
    Assert.assertEquals(selectionDataSchema.getColumnType(1), FieldSpec.DataType.INT);
    selectionResult = (Queue<Serializable[]>) resultsBlock.getSelectionResult();
    Assert.assertEquals(selectionResult.size(), 10);
    lastRow = selectionResult.peek();
    Assert.assertEquals(lastRow.length, 4);
    Assert.assertEquals(((Integer) lastRow[0]).intValue(), 6043515);
    Assert.assertEquals(((Integer) lastRow[1]).intValue(), 462769197);
}

Example 13 with ExecutionStatistics

use of com.linkedin.pinot.core.operator.ExecutionStatistics in project pinot by linkedin.

the class InnerSegmentSelectionSingleValueQueriesTest method testSelectStar.

@Test
public void testSelectStar() {
    String query = "SELECT * FROM testTable";
    // Test query without filter.
    MSelectionOnlyOperator selectionOnlyOperator = getOperatorForQuery(query);
    IntermediateResultsBlock resultsBlock = (IntermediateResultsBlock) selectionOnlyOperator.nextBlock();
    ExecutionStatistics executionStatistics = selectionOnlyOperator.getExecutionStatistics();
    Assert.assertEquals(executionStatistics.getNumDocsScanned(), 10L);
    Assert.assertEquals(executionStatistics.getNumEntriesScannedInFilter(), 0L);
    Assert.assertEquals(executionStatistics.getNumEntriesScannedPostFilter(), 110L);
    Assert.assertEquals(executionStatistics.getNumTotalRawDocs(), 30000L);
    DataSchema selectionDataSchema = resultsBlock.getSelectionDataSchema();
    Assert.assertEquals(selectionDataSchema.size(), 11);
    Assert.assertEquals(selectionDataSchema.getColumnName(0), "column1");
    Assert.assertEquals(selectionDataSchema.getColumnName(1), "column11");
    Assert.assertEquals(selectionDataSchema.getColumnType(0), FieldSpec.DataType.INT);
    Assert.assertEquals(selectionDataSchema.getColumnType(1), FieldSpec.DataType.STRING);
    List<Serializable[]> selectionResult = (List<Serializable[]>) resultsBlock.getSelectionResult();
    Assert.assertEquals(selectionResult.size(), 10);
    Serializable[] firstRow = selectionResult.get(0);
    Assert.assertEquals(firstRow.length, 11);
    Assert.assertEquals(((Integer) firstRow[0]).intValue(), 1578964907);
    Assert.assertEquals((String) firstRow[1], "P");
    // Test query with filter.
    selectionOnlyOperator = getOperatorForQueryWithFilter(query);
    resultsBlock = (IntermediateResultsBlock) selectionOnlyOperator.nextBlock();
    executionStatistics = selectionOnlyOperator.getExecutionStatistics();
    Assert.assertEquals(executionStatistics.getNumDocsScanned(), 10L);
    Assert.assertEquals(executionStatistics.getNumEntriesScannedInFilter(), 48241L);
    Assert.assertEquals(executionStatistics.getNumEntriesScannedPostFilter(), 110L);
    Assert.assertEquals(executionStatistics.getNumTotalRawDocs(), 30000L);
    selectionDataSchema = resultsBlock.getSelectionDataSchema();
    Assert.assertEquals(selectionDataSchema.size(), 11);
    Assert.assertEquals(selectionDataSchema.getColumnName(0), "column1");
    Assert.assertEquals(selectionDataSchema.getColumnName(1), "column11");
    Assert.assertEquals(selectionDataSchema.getColumnType(0), FieldSpec.DataType.INT);
    Assert.assertEquals(selectionDataSchema.getColumnType(1), FieldSpec.DataType.STRING);
    selectionResult = (List<Serializable[]>) resultsBlock.getSelectionResult();
    Assert.assertEquals(selectionResult.size(), 10);
    firstRow = selectionResult.get(0);
    Assert.assertEquals(firstRow.length, 11);
    Assert.assertEquals(((Integer) firstRow[0]).intValue(), 351823652);
    Assert.assertEquals((String) firstRow[1], "t");
}

Example 14 with ExecutionStatistics

use of com.linkedin.pinot.core.operator.ExecutionStatistics in project pinot by linkedin.

the class MSelectionOrderByOperator method getNextBlock.

@Override
public Block getNextBlock() {
    int numDocsScanned = 0;
    ProjectionBlock projectionBlock;
    while ((projectionBlock = (ProjectionBlock) _projectionOperator.nextBlock()) != null) {
        for (int i = 0; i < _dataSchema.size(); i++) {
            _blocks[i] = projectionBlock.getBlock(_dataSchema.getColumnName(i));
        }
        DocIdSetBlock docIdSetBlock = projectionBlock.getDocIdSetBlock();
        _selectionOperatorService.iterateOnBlocksWithOrdering(docIdSetBlock.getBlockDocIdSet().iterator(), _blocks);
    }
    // Create execution statistics.
    numDocsScanned += _selectionOperatorService.getNumDocsScanned();
    long numEntriesScannedInFilter = _projectionOperator.getExecutionStatistics().getNumEntriesScannedInFilter();
    long numEntriesScannedPostFilter = numDocsScanned * _projectionOperator.getNumProjectionColumns();
    long numTotalRawDocs = _indexSegment.getSegmentMetadata().getTotalRawDocs();
    _executionStatistics = new ExecutionStatistics(numDocsScanned, numEntriesScannedInFilter, numEntriesScannedPostFilter, numTotalRawDocs);
    return new IntermediateResultsBlock(_selectionOperatorService.getDataSchema(), _selectionOperatorService.getRows());
}

Example 15 with ExecutionStatistics

use of com.linkedin.pinot.core.operator.ExecutionStatistics in project pinot by linkedin.

the class InnerSegmentAggregationMultiValueQueriesTest method testSmallAggregationGroupBy.

@Test
public void testSmallAggregationGroupBy() {
    String query = "SELECT" + AGGREGATION + " FROM testTable" + SMALL_GROUP_BY;
    // NOTE: here we assume the first group key returned from the iterator is constant.
    // Test query without filter.
    AggregationGroupByOperator aggregationGroupByOperator = getOperatorForQuery(query);
    IntermediateResultsBlock resultsBlock = (IntermediateResultsBlock) aggregationGroupByOperator.nextBlock();
    ExecutionStatistics executionStatistics = aggregationGroupByOperator.getExecutionStatistics();
    Assert.assertEquals(executionStatistics.getNumDocsScanned(), 100000L);
    Assert.assertEquals(executionStatistics.getNumEntriesScannedInFilter(), 0L);
    Assert.assertEquals(executionStatistics.getNumEntriesScannedPostFilter(), 500000L);
    Assert.assertEquals(executionStatistics.getNumTotalRawDocs(), 100000L);
    AggregationGroupByResult aggregationGroupByResult = resultsBlock.getAggregationGroupByResult();
    GroupKeyGenerator.GroupKey firstGroupKey = aggregationGroupByResult.getGroupKeyIterator().next();
    Assert.assertEquals(firstGroupKey.getStringKey(), "201");
    Assert.assertEquals(((Number) aggregationGroupByResult.getResultForKey(firstGroupKey, 0)).longValue(), 26L);
    Assert.assertEquals(((Number) aggregationGroupByResult.getResultForKey(firstGroupKey, 1)).longValue(), 32555949195L);
    Assert.assertEquals(((Number) aggregationGroupByResult.getResultForKey(firstGroupKey, 2)).intValue(), 2100941020);
    Assert.assertEquals(((Number) aggregationGroupByResult.getResultForKey(firstGroupKey, 3)).intValue(), 117939666);
    AvgPair avgResult = (AvgPair) aggregationGroupByResult.getResultForKey(firstGroupKey, 4);
    Assert.assertEquals((long) avgResult.getSum(), 23061775005L);
    Assert.assertEquals(avgResult.getCount(), 26L);
    // Test query with filter.
    aggregationGroupByOperator = getOperatorForQueryWithFilter(query);
    resultsBlock = (IntermediateResultsBlock) aggregationGroupByOperator.nextBlock();
    executionStatistics = aggregationGroupByOperator.getExecutionStatistics();
    Assert.assertEquals(executionStatistics.getNumDocsScanned(), 15620L);
    Assert.assertEquals(executionStatistics.getNumEntriesScannedInFilter(), 282430L);
    Assert.assertEquals(executionStatistics.getNumEntriesScannedPostFilter(), 78100L);
    Assert.assertEquals(executionStatistics.getNumTotalRawDocs(), 100000L);
    aggregationGroupByResult = resultsBlock.getAggregationGroupByResult();
    firstGroupKey = aggregationGroupByResult.getGroupKeyIterator().next();
    Assert.assertEquals(firstGroupKey.getStringKey(), "203");
    Assert.assertEquals(((Number) aggregationGroupByResult.getResultForKey(firstGroupKey, 0)).longValue(), 1L);
    Assert.assertEquals(((Number) aggregationGroupByResult.getResultForKey(firstGroupKey, 1)).longValue(), 185436225L);
    Assert.assertEquals(((Number) aggregationGroupByResult.getResultForKey(firstGroupKey, 2)).intValue(), 987549258);
    Assert.assertEquals(((Number) aggregationGroupByResult.getResultForKey(firstGroupKey, 3)).intValue(), 674022574);
    avgResult = (AvgPair) aggregationGroupByResult.getResultForKey(firstGroupKey, 4);
    Assert.assertEquals((long) avgResult.getSum(), 674022574L);
    Assert.assertEquals(avgResult.getCount(), 1L);
}