Examples with DimensionSelector io.druid.segment.DimensionSelector used on opensource projects

Example 16 with DimensionSelector

use of io.druid.segment.DimensionSelector in project druid by druid-io.

the class SegmentAnalyzer method analyzeStringColumn.

private ColumnAnalysis analyzeStringColumn(final ColumnCapabilities capabilities, final StorageAdapter storageAdapter, final String columnName) {
    int cardinality = 0;
    long size = 0;
    Comparable min = null;
    Comparable max = null;
    if (analyzingCardinality()) {
        cardinality = storageAdapter.getDimensionCardinality(columnName);
    }
    if (analyzingSize()) {
        final long start = storageAdapter.getMinTime().getMillis();
        final long end = storageAdapter.getMaxTime().getMillis();
        final Sequence<Cursor> cursors = storageAdapter.makeCursors(null, new Interval(start, end), VirtualColumns.EMPTY, Granularities.ALL, false);
        size = cursors.accumulate(0L, new Accumulator<Long, Cursor>() {

            @Override
            public Long accumulate(Long accumulated, Cursor cursor) {
                DimensionSelector selector = cursor.makeDimensionSelector(new DefaultDimensionSpec(columnName, columnName));
                if (selector == null) {
                    return accumulated;
                }
                long current = accumulated;
                while (!cursor.isDone()) {
                    final IndexedInts vals = selector.getRow();
                    for (int i = 0; i < vals.size(); ++i) {
                        final String dimVal = selector.lookupName(vals.get(i));
                        if (dimVal != null && !dimVal.isEmpty()) {
                            current += StringUtils.estimatedBinaryLengthAsUTF8(dimVal);
                        }
                    }
                    cursor.advance();
                }
                return current;
            }
        });
    }
    if (analyzingMinMax()) {
        min = storageAdapter.getMinValue(columnName);
        max = storageAdapter.getMaxValue(columnName);
    }
    return new ColumnAnalysis(capabilities.getType().name(), capabilities.hasMultipleValues(), size, cardinality, min, max, null);
}

Example 17 with DimensionSelector

use of io.druid.segment.DimensionSelector in project druid by druid-io.

the class CardinalityAggregatorBenchmark method setUp.

protected void setUp() {
    Iterable<String[]> values = FluentIterable.from(ContiguousSet.create(Range.closedOpen(0, 500), DiscreteDomain.integers())).transform(new Function<Integer, String[]>() {

        @Override
        public String[] apply(Integer input) {
            if (multivaluedSized == 1) {
                return new String[] { input.toString() };
            } else {
                String[] res = new String[multivaluedSized];
                String value = input.toString();
                for (int i = 0; i < multivaluedSized; ++i) {
                    res[i] = value + i;
                }
                return res;
            }
        }
    }).cycle().limit(MAX);
    final DimensionSpec dimSpec1 = new DefaultDimensionSpec("dim1", "dim1");
    final CardinalityAggregatorTest.TestDimensionSelector dim1 = new CardinalityAggregatorTest.TestDimensionSelector(values, null);
    final ColumnSelectorPlus<CardinalityAggregatorColumnSelectorStrategy> dimInfo1 = new ColumnSelectorPlus(dimSpec1.getDimension(), dimSpec1.getOutputName(), new StringCardinalityAggregatorColumnSelectorStrategy(), dim1);
    selectorList = Lists.newArrayList((DimensionSelector) dim1);
    dimInfos = new ColumnSelectorPlus[] { dimInfo1 };
    agg = new CardinalityBufferAggregator(dimInfos, byRow);
    CardinalityAggregatorFactory factory = new CardinalityAggregatorFactory("billy", Lists.<DimensionSpec>newArrayList(new DefaultDimensionSpec("dim1", "dim1")), byRow);
    int maxSize = factory.getMaxIntermediateSize();
    buf = ByteBuffer.allocate(maxSize + 64);
    pos = 10;
    buf.limit(pos + maxSize);
    agg.init(buf, pos);
}

Example 18 with DimensionSelector

use of io.druid.segment.DimensionSelector in project druid by druid-io.

the class IncrementalIndexReadBenchmark method readWithFilters.

@Benchmark
@BenchmarkMode(Mode.AverageTime)
@OutputTimeUnit(TimeUnit.MICROSECONDS)
public void readWithFilters(Blackhole blackhole) throws Exception {
    DimFilter filter = new OrDimFilter(Arrays.asList(new BoundDimFilter("dimSequential", "-1", "-1", true, true, null, null, StringComparators.ALPHANUMERIC), new JavaScriptDimFilter("dimSequential", "function(x) { return false }", null, JavaScriptConfig.getEnabledInstance()), new RegexDimFilter("dimSequential", "X", null), new SearchQueryDimFilter("dimSequential", new ContainsSearchQuerySpec("X", false), null), new InDimFilter("dimSequential", Arrays.asList("X"), null)));
    IncrementalIndexStorageAdapter sa = new IncrementalIndexStorageAdapter(incIndex);
    Sequence<Cursor> cursors = makeCursors(sa, filter);
    Cursor cursor = Sequences.toList(Sequences.limit(cursors, 1), Lists.<Cursor>newArrayList()).get(0);
    List<DimensionSelector> selectors = new ArrayList<>();
    selectors.add(cursor.makeDimensionSelector(new DefaultDimensionSpec("dimSequential", null)));
    selectors.add(cursor.makeDimensionSelector(new DefaultDimensionSpec("dimZipf", null)));
    selectors.add(cursor.makeDimensionSelector(new DefaultDimensionSpec("dimUniform", null)));
    selectors.add(cursor.makeDimensionSelector(new DefaultDimensionSpec("dimSequentialHalfNull", null)));
    cursor.reset();
    while (!cursor.isDone()) {
        for (DimensionSelector selector : selectors) {
            IndexedInts row = selector.getRow();
            blackhole.consume(selector.lookupName(row.get(0)));
        }
        cursor.advance();
    }
}

Example 19 with DimensionSelector

use of io.druid.segment.DimensionSelector in project druid by druid-io.

the class BaseFilterTest method selectColumnValuesMatchingFilterUsingPostFiltering.

private List<String> selectColumnValuesMatchingFilterUsingPostFiltering(final DimFilter filter, final String selectColumn) {
    final Filter theFilter = makeFilter(filter);
    final Filter postFilteringFilter = new Filter() {

        @Override
        public ImmutableBitmap getBitmapIndex(BitmapIndexSelector selector) {
            throw new UnsupportedOperationException();
        }

        @Override
        public ValueMatcher makeMatcher(ColumnSelectorFactory factory) {
            return theFilter.makeMatcher(factory);
        }

        @Override
        public boolean supportsBitmapIndex(BitmapIndexSelector selector) {
            return false;
        }

        @Override
        public boolean supportsSelectivityEstimation(ColumnSelector columnSelector, BitmapIndexSelector indexSelector) {
            return false;
        }

        @Override
        public double estimateSelectivity(BitmapIndexSelector indexSelector) {
            return 1.0;
        }
    };
    final Sequence<Cursor> cursors = makeCursorSequence(postFilteringFilter);
    Sequence<List<String>> seq = Sequences.map(cursors, new Function<Cursor, List<String>>() {

        @Override
        public List<String> apply(Cursor input) {
            final DimensionSelector selector = input.makeDimensionSelector(new DefaultDimensionSpec(selectColumn, selectColumn));
            final List<String> values = Lists.newArrayList();
            while (!input.isDone()) {
                IndexedInts row = selector.getRow();
                Preconditions.checkState(row.size() == 1);
                values.add(selector.lookupName(row.get(0)));
                input.advance();
            }
            return values;
        }
    });
    return Sequences.toList(seq, new ArrayList<List<String>>()).get(0);
}

Example 20 with DimensionSelector

use of io.druid.segment.DimensionSelector in project druid by druid-io.

the class IncrementalIndexStorageAdapterTest method testResetSanity.

@Test
public void testResetSanity() throws IOException {
    IncrementalIndex index = indexCreator.createIndex();
    DateTime t = DateTime.now();
    Interval interval = new Interval(t.minusMinutes(1), t.plusMinutes(1));
    index.add(new MapBasedInputRow(t.minus(1).getMillis(), Lists.newArrayList("billy"), ImmutableMap.<String, Object>of("billy", "hi")));
    index.add(new MapBasedInputRow(t.minus(1).getMillis(), Lists.newArrayList("sally"), ImmutableMap.<String, Object>of("sally", "bo")));
    IncrementalIndexStorageAdapter adapter = new IncrementalIndexStorageAdapter(index);
    for (boolean descending : Arrays.asList(false, true)) {
        Sequence<Cursor> cursorSequence = adapter.makeCursors(new SelectorFilter("sally", "bo"), interval, VirtualColumns.EMPTY, Granularities.NONE, descending);
        Cursor cursor = Sequences.toList(Sequences.limit(cursorSequence, 1), Lists.<Cursor>newArrayList()).get(0);
        DimensionSelector dimSelector;
        dimSelector = cursor.makeDimensionSelector(new DefaultDimensionSpec("sally", "sally"));
        Assert.assertEquals("bo", dimSelector.lookupName(dimSelector.getRow().get(0)));
        index.add(new MapBasedInputRow(t.minus(1).getMillis(), Lists.newArrayList("sally"), ImmutableMap.<String, Object>of("sally", "ah")));
        // Cursor reset should not be affected by out of order values
        cursor.reset();
        dimSelector = cursor.makeDimensionSelector(new DefaultDimensionSpec("sally", "sally"));
        Assert.assertEquals("bo", dimSelector.lookupName(dimSelector.getRow().get(0)));
    }
}