Examples with QueryableIndex org.apache.druid.segment.QueryableIndex used on opensource projects

Example 41 with QueryableIndex

use of org.apache.druid.segment.QueryableIndex in project druid by druid-io.

the class ThetaSketchSqlAggregatorTest method createQuerySegmentWalker.

@Override
public SpecificSegmentsQuerySegmentWalker createQuerySegmentWalker() throws IOException {
    SketchModule.registerSerde();
    final QueryableIndex index = IndexBuilder.create().tmpDir(temporaryFolder.newFolder()).segmentWriteOutMediumFactory(OffHeapMemorySegmentWriteOutMediumFactory.instance()).schema(new IncrementalIndexSchema.Builder().withMetrics(new CountAggregatorFactory("cnt"), new DoubleSumAggregatorFactory("m1", "m1"), new SketchMergeAggregatorFactory("thetasketch_dim1", "dim1", null, false, false, null)).withRollup(false).build()).rows(CalciteTests.ROWS1).buildMMappedIndex();
    return new SpecificSegmentsQuerySegmentWalker(conglomerate).add(DataSegment.builder().dataSource(DATA_SOURCE).interval(index.getDataInterval()).version("1").shardSpec(new LinearShardSpec(0)).size(0).build(), index);
}

Example 42 with QueryableIndex

use of org.apache.druid.segment.QueryableIndex in project druid by druid-io.

the class SegmentAnalyzer method analyze.

public Map<String, ColumnAnalysis> analyze(Segment segment) {
    Preconditions.checkNotNull(segment, "segment");
    // index is null for incremental-index-based segments, but storageAdapter is always available
    final QueryableIndex index = segment.asQueryableIndex();
    final StorageAdapter storageAdapter = segment.asStorageAdapter();
    // get length and column names from storageAdapter
    final int length = storageAdapter.getNumRows();
    Map<String, ColumnAnalysis> columns = new TreeMap<>();
    final RowSignature rowSignature = storageAdapter.getRowSignature();
    for (String columnName : rowSignature.getColumnNames()) {
        final ColumnCapabilities capabilities;
        if (storageAdapter instanceof IncrementalIndexStorageAdapter) {
            // See javadocs for getSnapshotColumnCapabilities for a discussion of why we need to do this.
            capabilities = ((IncrementalIndexStorageAdapter) storageAdapter).getSnapshotColumnCapabilities(columnName);
        } else {
            capabilities = storageAdapter.getColumnCapabilities(columnName);
        }
        final ColumnAnalysis analysis;
        switch(capabilities.getType()) {
            case LONG:
                final int bytesPerRow = ColumnHolder.TIME_COLUMN_NAME.equals(columnName) ? NUM_BYTES_IN_TIMESTAMP : Long.BYTES;
                analysis = analyzeNumericColumn(capabilities, length, bytesPerRow);
                break;
            case FLOAT:
                analysis = analyzeNumericColumn(capabilities, length, NUM_BYTES_IN_TEXT_FLOAT);
                break;
            case DOUBLE:
                analysis = analyzeNumericColumn(capabilities, length, Double.BYTES);
                break;
            case STRING:
                if (index != null) {
                    analysis = analyzeStringColumn(capabilities, index.getColumnHolder(columnName));
                } else {
                    analysis = analyzeStringColumn(capabilities, storageAdapter, columnName);
                }
                break;
            case COMPLEX:
                final ColumnHolder columnHolder = index != null ? index.getColumnHolder(columnName) : null;
                analysis = analyzeComplexColumn(capabilities, columnHolder);
                break;
            default:
                log.warn("Unknown column type[%s].", capabilities.asTypeString());
                analysis = ColumnAnalysis.error(StringUtils.format("unknown_type_%s", capabilities.asTypeString()));
        }
        columns.put(columnName, analysis);
    }
    return columns;
}

Example 43 with QueryableIndex

use of org.apache.druid.segment.QueryableIndex in project druid by druid-io.

the class AutoStrategy method getExecutionPlan.

@Override
public List<SearchQueryExecutor> getExecutionPlan(SearchQuery query, Segment segment) {
    final QueryableIndex index = segment.asQueryableIndex();
    if (index != null) {
        final BitmapIndexSelector selector = new ColumnSelectorBitmapIndexSelector(index.getBitmapFactoryForDimensions(), VirtualColumns.EMPTY, index);
        // from the non-bitmap-support filters, and then use it to compute the filtered result by intersecting bitmaps.
        if (filter == null || filter.supportsSelectivityEstimation(index, selector)) {
            final List<DimensionSpec> dimsToSearch = getDimsToSearch(index.getAvailableDimensions(), query.getDimensions());
            // Choose a search query execution strategy depending on the query.
            // The costs of index-only plan and cursor-based plan can be computed like below.
            // 
            // c_index = (total cardinality of all search dimensions) * (bitmap intersection cost)
            // * (search predicate processing cost)
            // c_cursor = (# of rows in a segment) * (filter selectivity) * (# of dimensions)
            // * (search predicate processing cost)
            final SearchQueryDecisionHelper helper = getDecisionHelper(index);
            final double useIndexStrategyCost = helper.getBitmapIntersectCost() * computeTotalCard(index, dimsToSearch);
            final double cursorOnlyStrategyCost = (filter == null ? 1. : filter.estimateSelectivity(selector)) * selector.getNumRows() * dimsToSearch.size();
            log.debug("Use-index strategy cost: %f, cursor-only strategy cost: %f", useIndexStrategyCost, cursorOnlyStrategyCost);
            if (useIndexStrategyCost < cursorOnlyStrategyCost) {
                log.debug("Use-index execution strategy is selected, query id [%s]", query.getId());
                return UseIndexesStrategy.of(query).getExecutionPlan(query, segment);
            } else {
                log.debug("Cursor-only execution strategy is selected, query id [%s]", query.getId());
                return CursorOnlyStrategy.of(query).getExecutionPlan(query, segment);
            }
        } else {
            log.debug("Filter doesn't support bitmap index. Fall back to cursor-only execution strategy, query id [%s]", query.getId());
            return CursorOnlyStrategy.of(query).getExecutionPlan(query, segment);
        }
    } else {
        log.debug("Index doesn't exist. Fall back to cursor-only execution strategy, query id [%s]", query.getId());
        return CursorOnlyStrategy.of(query).getExecutionPlan(query, segment);
    }
}

Example 44 with QueryableIndex

use of org.apache.druid.segment.QueryableIndex in project druid by druid-io.

the class GroupByLimitPushDownInsufficientBufferTest method getRunner1.

private List<QueryRunner<ResultRow>> getRunner1() {
    List<QueryRunner<ResultRow>> runners = new ArrayList<>();
    QueryableIndex index = groupByIndices.get(0);
    QueryRunner<ResultRow> runner = makeQueryRunner(groupByFactory, SegmentId.dummy(index.toString()), new QueryableIndexSegment(index, SegmentId.dummy(index.toString())));
    runners.add(groupByFactory.getToolchest().preMergeQueryDecoration(runner));
    return runners;
}

Example 45 with QueryableIndex

use of org.apache.druid.segment.QueryableIndex in project druid by druid-io.

the class GroupByMultiSegmentTest method setup.

@Before
public void setup() throws Exception {
    tmpDir = FileUtils.createTempDir();
    InputRow row;
    List<String> dimNames = Arrays.asList("dimA", "metA");
    Map<String, Object> event;
    final IncrementalIndex indexA = makeIncIndex(false);
    incrementalIndices.add(indexA);
    event = new HashMap<>();
    event.put("dimA", "hello");
    event.put("metA", 100);
    row = new MapBasedInputRow(1000, dimNames, event);
    indexA.add(row);
    event = new HashMap<>();
    event.put("dimA", "world");
    event.put("metA", 75);
    row = new MapBasedInputRow(1000, dimNames, event);
    indexA.add(row);
    final File fileA = INDEX_MERGER_V9.persist(indexA, new File(tmpDir, "A"), new IndexSpec(), null);
    QueryableIndex qindexA = INDEX_IO.loadIndex(fileA);
    final IncrementalIndex indexB = makeIncIndex(false);
    incrementalIndices.add(indexB);
    event = new HashMap<>();
    event.put("dimA", "foo");
    event.put("metA", 100);
    row = new MapBasedInputRow(1000, dimNames, event);
    indexB.add(row);
    event = new HashMap<>();
    event.put("dimA", "world");
    event.put("metA", 75);
    row = new MapBasedInputRow(1000, dimNames, event);
    indexB.add(row);
    final File fileB = INDEX_MERGER_V9.persist(indexB, new File(tmpDir, "B"), new IndexSpec(), null);
    QueryableIndex qindexB = INDEX_IO.loadIndex(fileB);
    groupByIndices = Arrays.asList(qindexA, qindexB);
    resourceCloser = Closer.create();
    setupGroupByFactory();
}