Examples with DataSource com.linkedin.pinot.core.common.DataSource used on opensource projects

Example 21 with DataSource

use of com.linkedin.pinot.core.common.DataSource in project pinot by linkedin.

the class StarTreeIndexOperator method createChildOperator.

private BaseFilterOperator createChildOperator(int startDocId, int endDocId, String column, PredicateEntry predicateEntry) {
    DataSource dataSource = segment.getDataSource(column);
    DataSourceMetadata dataSourceMetadata = dataSource.getDataSourceMetadata();
    BaseFilterOperator childOperator;
    Predicate predicate = predicateEntry.predicate;
    if (dataSourceMetadata.hasInvertedIndex()) {
        if (dataSourceMetadata.isSorted()) {
            childOperator = new SortedInvertedIndexBasedFilterOperator(predicate, dataSource, startDocId, endDocId);
        } else {
            childOperator = new BitmapBasedFilterOperator(predicate, dataSource, startDocId, endDocId);
        }
    } else {
        childOperator = new ScanBasedFilterOperator(predicate, dataSource, startDocId, endDocId);
    }
    return childOperator;
}

Example 22 with DataSource

use of com.linkedin.pinot.core.common.DataSource in project pinot by linkedin.

the class BaseSumStarTreeIndexTest method computeSum.

/**
   * Compute 'sum' for a given list of metrics, by scanning the given set of doc-ids.
   *
   * @param segment
   * @param docIdIterator
   * @param metricNames
   * @return
   */
private Map<String, double[]> computeSum(IndexSegment segment, BlockDocIdIterator docIdIterator, List<String> metricNames, List<String> groupByColumns) {
    int docId;
    int numMetrics = metricNames.size();
    Dictionary[] metricDictionaries = new Dictionary[numMetrics];
    BlockSingleValIterator[] metricValIterators = new BlockSingleValIterator[numMetrics];
    int numGroupByColumns = groupByColumns.size();
    Dictionary[] groupByDictionaries = new Dictionary[numGroupByColumns];
    BlockSingleValIterator[] groupByValIterators = new BlockSingleValIterator[numGroupByColumns];
    for (int i = 0; i < numMetrics; i++) {
        String metricName = metricNames.get(i);
        DataSource dataSource = segment.getDataSource(metricName);
        metricDictionaries[i] = dataSource.getDictionary();
        metricValIterators[i] = (BlockSingleValIterator) dataSource.getNextBlock().getBlockValueSet().iterator();
    }
    for (int i = 0; i < numGroupByColumns; i++) {
        String groupByColumn = groupByColumns.get(i);
        DataSource dataSource = segment.getDataSource(groupByColumn);
        groupByDictionaries[i] = dataSource.getDictionary();
        groupByValIterators[i] = (BlockSingleValIterator) dataSource.getNextBlock().getBlockValueSet().iterator();
    }
    Map<String, double[]> result = new HashMap<String, double[]>();
    while ((docId = docIdIterator.next()) != Constants.EOF) {
        StringBuilder stringBuilder = new StringBuilder();
        for (int i = 0; i < numGroupByColumns; i++) {
            groupByValIterators[i].skipTo(docId);
            int dictId = groupByValIterators[i].nextIntVal();
            stringBuilder.append(groupByDictionaries[i].getStringValue(dictId));
            stringBuilder.append("_");
        }
        String key = stringBuilder.toString();
        if (!result.containsKey(key)) {
            result.put(key, new double[numMetrics]);
        }
        double[] sumsSoFar = result.get(key);
        for (int i = 0; i < numMetrics; i++) {
            metricValIterators[i].skipTo(docId);
            int dictId = metricValIterators[i].nextIntVal();
            sumsSoFar[i] += metricDictionaries[i].getDoubleValue(dictId);
        }
    }
    return result;
}

Example 23 with DataSource

use of com.linkedin.pinot.core.common.DataSource in project pinot by linkedin.

the class RealtimeFileBasedReaderTest method testDataSourceWithoutPredicateForMultiValueDimensionColumns.

private void testDataSourceWithoutPredicateForMultiValueDimensionColumns() {
    for (FieldSpec spec : schema.getAllFieldSpecs()) {
        if (!spec.isSingleValueField()) {
            DataSource offlineDS = offlineSegment.getDataSource(spec.getName());
            DataSource realtimeDS = realtimeSegment.getDataSource(spec.getName());
            Block offlineBlock = offlineDS.nextBlock();
            Block realtimeBlock = realtimeDS.nextBlock();
            BlockMetadata offlineMetadata = offlineBlock.getMetadata();
            BlockMetadata realtimeMetadata = realtimeBlock.getMetadata();
            BlockMultiValIterator offlineValIterator = (BlockMultiValIterator) offlineBlock.getBlockValueSet().iterator();
            BlockMultiValIterator realtimeValIterator = (BlockMultiValIterator) realtimeBlock.getBlockValueSet().iterator();
            Assert.assertEquals(offlineSegment.getSegmentMetadata().getTotalDocs(), realtimeSegment.getAggregateDocumentCount());
            while (realtimeValIterator.hasNext()) {
                int[] offlineIds = new int[offlineBlock.getMetadata().getMaxNumberOfMultiValues()];
                int[] realtimeIds = new int[realtimeBlock.getMetadata().getMaxNumberOfMultiValues()];
                int Olen = offlineValIterator.nextIntVal(offlineIds);
                int Rlen = realtimeValIterator.nextIntVal(realtimeIds);
                Assert.assertEquals(Olen, Rlen);
                for (int i = 0; i < Olen; i++) {
                    Assert.assertEquals(offlineMetadata.getDictionary().get(offlineIds[i]), realtimeMetadata.getDictionary().get(realtimeIds[i]));
                }
            }
        }
    }
}

Example 24 with DataSource

use of com.linkedin.pinot.core.common.DataSource in project pinot by linkedin.

the class RealtimeSegmentTest method testNoMatchFilteringMetricPredicateWithoutInvIdx.

@Test
public void testNoMatchFilteringMetricPredicateWithoutInvIdx() throws Exception {
    DataSource ds1 = segmentWithoutInvIdx.getDataSource("count");
    List<String> rhs = new ArrayList<String>();
    rhs.add("890662862");
    Predicate predicate = new NEqPredicate("count", rhs);
    ScanBasedFilterOperator op = new ScanBasedFilterOperator(predicate, ds1, 0, segmentWithoutInvIdx.getRawDocumentCount() - 1);
    Block b = op.nextBlock();
    BlockDocIdIterator iterator = b.getBlockDocIdSet().iterator();
    int counter = 0;
    int docId = iterator.next();
    while (docId != Constants.EOF) {
        // shouldn't reach here.
        Assert.assertTrue(false);
        docId = iterator.next();
        counter++;
    }
    Assert.assertEquals(counter, 0);
}

Example 25 with DataSource

use of com.linkedin.pinot.core.common.DataSource in project pinot by linkedin.

the class RealtimeSegmentTest method testNoRangeMatchFilteringMetricPredicateWithInvIdx.

@Test
public void testNoRangeMatchFilteringMetricPredicateWithInvIdx() throws Exception {
    DataSource ds1 = segmentWithInvIdx.getDataSource("count");
    List<String> rhs = new ArrayList<String>();
    rhs.add("[0\t\t100)");
    Predicate predicate = new RangePredicate("count", rhs);
    BitmapBasedFilterOperator op = new BitmapBasedFilterOperator(predicate, ds1, 0, segmentWithInvIdx.getRawDocumentCount() - 1);
    Block b = op.nextBlock();
    BlockDocIdIterator iterator = b.getBlockDocIdSet().iterator();
    int counter = 0;
    int docId = iterator.next();
    while (docId != Constants.EOF) {
        // shouldn't reach here.
        Assert.assertTrue(false);
        docId = iterator.next();
        counter++;
    }
    Assert.assertEquals(counter, 0);
}