Examples with IndexedInts io.druid.segment.data.IndexedInts used on opensource projects

Example 31 with IndexedInts

use of io.druid.segment.data.IndexedInts in project druid by druid-io.

the class QueryableIndexIndexableAdapterTest method testGetBitmapIndex.

@Test
public void testGetBitmapIndex() throws Exception {
    final long timestamp = System.currentTimeMillis();
    IncrementalIndex toPersist = IncrementalIndexTest.createIndex(null);
    IncrementalIndexTest.populateIndex(timestamp, toPersist);
    final File tempDir = temporaryFolder.newFolder();
    QueryableIndex index = closer.closeLater(INDEX_IO.loadIndex(INDEX_MERGER.persist(toPersist, tempDir, INDEX_SPEC)));
    IndexableAdapter adapter = new QueryableIndexIndexableAdapter(index);
    String dimension = "dim1";
    //null is added to all dimensions with value
    IndexedInts indexedInts = adapter.getBitmapIndex(dimension, 0);
    for (int i = 0; i < adapter.getDimValueLookup(dimension).size(); i++) {
        indexedInts = adapter.getBitmapIndex(dimension, i);
        Assert.assertEquals(1, indexedInts.size());
    }
}

Example 32 with IndexedInts

use of io.druid.segment.data.IndexedInts in project druid by druid-io.

the class DictionaryEncodedColumnPartSerde method getDeserializer.

@Override
public Deserializer getDeserializer() {
    return new Deserializer() {

        @Override
        public void read(ByteBuffer buffer, ColumnBuilder builder, ColumnConfig columnConfig) {
            final VERSION rVersion = VERSION.fromByte(buffer.get());
            final int rFlags;
            if (rVersion.compareTo(VERSION.COMPRESSED) >= 0) {
                rFlags = buffer.getInt();
            } else {
                rFlags = rVersion.equals(VERSION.UNCOMPRESSED_MULTI_VALUE) ? Feature.MULTI_VALUE.getMask() : NO_FLAGS;
            }
            final boolean hasMultipleValues = Feature.MULTI_VALUE.isSet(rFlags) || Feature.MULTI_VALUE_V3.isSet(rFlags);
            final GenericIndexed<String> rDictionary = GenericIndexed.read(buffer, GenericIndexed.STRING_STRATEGY, builder.getFileMapper());
            builder.setType(ValueType.STRING);
            final WritableSupplier<IndexedInts> rSingleValuedColumn;
            final WritableSupplier<IndexedMultivalue<IndexedInts>> rMultiValuedColumn;
            if (hasMultipleValues) {
                rMultiValuedColumn = readMultiValuedColum(rVersion, buffer, rFlags, builder.getFileMapper());
                rSingleValuedColumn = null;
            } else {
                rSingleValuedColumn = readSingleValuedColumn(rVersion, buffer, builder.getFileMapper());
                rMultiValuedColumn = null;
            }
            builder.setHasMultipleValues(hasMultipleValues).setDictionaryEncodedColumn(new DictionaryEncodedColumnSupplier(rDictionary, rSingleValuedColumn, rMultiValuedColumn, columnConfig.columnCacheSizeBytes()));
            GenericIndexed<ImmutableBitmap> rBitmaps = GenericIndexed.read(buffer, bitmapSerdeFactory.getObjectStrategy(), builder.getFileMapper());
            builder.setBitmapIndex(new BitmapIndexColumnPartSupplier(bitmapSerdeFactory.getBitmapFactory(), rBitmaps, rDictionary));
            ImmutableRTree rSpatialIndex = null;
            if (buffer.hasRemaining()) {
                rSpatialIndex = ByteBufferSerializer.read(buffer, new IndexedRTree.ImmutableRTreeObjectStrategy(bitmapSerdeFactory.getBitmapFactory()));
                builder.setSpatialIndex(new SpatialIndexColumnPartSupplier(rSpatialIndex));
            }
        }

        private WritableSupplier<IndexedInts> readSingleValuedColumn(VERSION version, ByteBuffer buffer, SmooshedFileMapper fileMapper) {
            switch(version) {
                case UNCOMPRESSED_SINGLE_VALUE:
                    return VSizeIndexedInts.readFromByteBuffer(buffer).asWritableSupplier();
                case COMPRESSED:
                    return CompressedVSizeIntsIndexedSupplier.fromByteBuffer(buffer, byteOrder, fileMapper);
            }
            throw new IAE("Unsupported single-value version[%s]", version);
        }

        private WritableSupplier<IndexedMultivalue<IndexedInts>> readMultiValuedColum(VERSION version, ByteBuffer buffer, int flags, SmooshedFileMapper fileMapper) {
            switch(version) {
                case UNCOMPRESSED_MULTI_VALUE:
                    return VSizeIndexed.readFromByteBuffer(buffer).asWritableSupplier();
                case COMPRESSED:
                    if (Feature.MULTI_VALUE.isSet(flags)) {
                        return CompressedVSizeIndexedSupplier.fromByteBuffer(buffer, byteOrder, fileMapper);
                    } else if (Feature.MULTI_VALUE_V3.isSet(flags)) {
                        return CompressedVSizeIndexedV3Supplier.fromByteBuffer(buffer, byteOrder, fileMapper);
                    } else {
                        throw new IAE("Unrecognized multi-value flag[%d]", flags);
                    }
            }
            throw new IAE("Unsupported multi-value version[%s]", version);
        }
    };
}

Example 33 with IndexedInts

use of io.druid.segment.data.IndexedInts in project druid by druid-io.

the class BaseFilterTest method selectColumnValuesMatchingFilterUsingPostFiltering.

private List<String> selectColumnValuesMatchingFilterUsingPostFiltering(final DimFilter filter, final String selectColumn) {
    final Filter theFilter = makeFilter(filter);
    final Filter postFilteringFilter = new Filter() {

        @Override
        public ImmutableBitmap getBitmapIndex(BitmapIndexSelector selector) {
            throw new UnsupportedOperationException();
        }

        @Override
        public ValueMatcher makeMatcher(ColumnSelectorFactory factory) {
            return theFilter.makeMatcher(factory);
        }

        @Override
        public boolean supportsBitmapIndex(BitmapIndexSelector selector) {
            return false;
        }

        @Override
        public boolean supportsSelectivityEstimation(ColumnSelector columnSelector, BitmapIndexSelector indexSelector) {
            return false;
        }

        @Override
        public double estimateSelectivity(BitmapIndexSelector indexSelector) {
            return 1.0;
        }
    };
    final Sequence<Cursor> cursors = makeCursorSequence(postFilteringFilter);
    Sequence<List<String>> seq = Sequences.map(cursors, new Function<Cursor, List<String>>() {

        @Override
        public List<String> apply(Cursor input) {
            final DimensionSelector selector = input.makeDimensionSelector(new DefaultDimensionSpec(selectColumn, selectColumn));
            final List<String> values = Lists.newArrayList();
            while (!input.isDone()) {
                IndexedInts row = selector.getRow();
                Preconditions.checkState(row.size() == 1);
                values.add(selector.lookupName(row.get(0)));
                input.advance();
            }
            return values;
        }
    });
    return Sequences.toList(seq, new ArrayList<List<String>>()).get(0);
}

Example 34 with IndexedInts

use of io.druid.segment.data.IndexedInts in project druid by druid-io.

the class MapVirtualColumn method makeObjectColumnSelector.

@Override
public ObjectColumnSelector makeObjectColumnSelector(String dimension, ColumnSelectorFactory factory) {
    final DimensionSelector keySelector = factory.makeDimensionSelector(DefaultDimensionSpec.of(keyDimension));
    final DimensionSelector valueSelector = factory.makeDimensionSelector(DefaultDimensionSpec.of(valueDimension));
    final String subColumnName = VirtualColumns.splitColumnName(dimension).rhs;
    if (subColumnName == null) {
        return new ObjectColumnSelector<Map>() {

            @Override
            public Class classOfObject() {
                return Map.class;
            }

            @Override
            public Map get() {
                final IndexedInts keyIndices = keySelector.getRow();
                final IndexedInts valueIndices = valueSelector.getRow();
                if (keyIndices == null || valueIndices == null) {
                    return null;
                }
                final int limit = Math.min(keyIndices.size(), valueIndices.size());
                final Map<String, String> map = Maps.newHashMapWithExpectedSize(limit);
                for (int i = 0; i < limit; i++) {
                    map.put(keySelector.lookupName(keyIndices.get(i)), valueSelector.lookupName(valueIndices.get(i)));
                }
                return map;
            }
        };
    }
    IdLookup keyIdLookup = keySelector.idLookup();
    if (keyIdLookup != null) {
        final int keyId = keyIdLookup.lookupId(subColumnName);
        if (keyId < 0) {
            return NullStringObjectColumnSelector.instance();
        }
        return new ObjectColumnSelector<String>() {

            @Override
            public Class classOfObject() {
                return String.class;
            }

            @Override
            public String get() {
                final IndexedInts keyIndices = keySelector.getRow();
                final IndexedInts valueIndices = valueSelector.getRow();
                if (keyIndices == null || valueIndices == null) {
                    return null;
                }
                final int limit = Math.min(keyIndices.size(), valueIndices.size());
                for (int i = 0; i < limit; i++) {
                    if (keyIndices.get(i) == keyId) {
                        return valueSelector.lookupName(valueIndices.get(i));
                    }
                }
                return null;
            }
        };
    } else {
        return new ObjectColumnSelector<String>() {

            @Override
            public Class classOfObject() {
                return String.class;
            }

            @Override
            public String get() {
                final IndexedInts keyIndices = keySelector.getRow();
                final IndexedInts valueIndices = valueSelector.getRow();
                if (keyIndices == null || valueIndices == null) {
                    return null;
                }
                final int limit = Math.min(keyIndices.size(), valueIndices.size());
                for (int i = 0; i < limit; i++) {
                    if (Objects.equals(keySelector.lookupName(keyIndices.get(i)), subColumnName)) {
                        return valueSelector.lookupName(valueIndices.get(i));
                    }
                }
                return null;
            }
        };
    }
}

Example 35 with IndexedInts

use of io.druid.segment.data.IndexedInts in project druid by druid-io.

the class ListFilteredDimensionSpecTest method testDecoratorWithWhitelist.

@Test
public void testDecoratorWithWhitelist() {
    ListFilteredDimensionSpec spec = new ListFilteredDimensionSpec(new DefaultDimensionSpec("foo", "bar"), ImmutableSet.of("c", "g"), true);
    DimensionSelector selector = spec.decorate(TestDimensionSelector.instance);
    Assert.assertEquals(2, selector.getValueCardinality());
    IndexedInts row = selector.getRow();
    Assert.assertEquals(2, row.size());
    Assert.assertEquals(0, row.get(0));
    Assert.assertEquals(1, row.get(1));
    Assert.assertEquals("c", selector.lookupName(0));
    Assert.assertEquals("g", selector.lookupName(1));
    Assert.assertEquals(0, selector.idLookup().lookupId("c"));
    Assert.assertEquals(1, selector.idLookup().lookupId("g"));
}