Examples with ObjectColumnSelector io.druid.segment.ObjectColumnSelector used on opensource projects

Example 11 with ObjectColumnSelector

use of io.druid.segment.ObjectColumnSelector in project druid by druid-io.

the class DumpSegment method makeSelector.

private static ObjectColumnSelector makeSelector(final String columnName, final Column column, final ColumnSelectorFactory columnSelectorFactory) {
    final ObjectColumnSelector selector;
    if (column.getDictionaryEncoding() != null) {
        // Special case for dimensions -> always wrap multi-value in arrays
        final DimensionSelector dimensionSelector = columnSelectorFactory.makeDimensionSelector(new DefaultDimensionSpec(columnName, columnName));
        if (column.getDictionaryEncoding().hasMultipleValues()) {
            return new ObjectColumnSelector<List>() {

                @Override
                public Class<List> classOfObject() {
                    return List.class;
                }

                @Override
                public List<String> get() {
                    final IndexedInts row = dimensionSelector.getRow();
                    if (row.size() == 0) {
                        return null;
                    } else {
                        final List<String> retVal = Lists.newArrayList();
                        for (int i = 0; i < row.size(); i++) {
                            retVal.add(dimensionSelector.lookupName(row.get(i)));
                        }
                        return retVal;
                    }
                }
            };
        } else {
            return new ObjectColumnSelector<String>() {

                @Override
                public Class<String> classOfObject() {
                    return String.class;
                }

                @Override
                public String get() {
                    final IndexedInts row = dimensionSelector.getRow();
                    return row.size() == 0 ? null : dimensionSelector.lookupName(row.get(0));
                }
            };
        }
    } else {
        final ObjectColumnSelector maybeSelector = columnSelectorFactory.makeObjectColumnSelector(columnName);
        if (maybeSelector != null) {
            selector = maybeSelector;
        } else {
            // Selector failed to create (unrecognized column type?)
            log.warn("Could not create selector for column[%s], returning null.", columnName);
            selector = new ObjectColumnSelector() {

                @Override
                public Class classOfObject() {
                    return Object.class;
                }

                @Override
                public Object get() {
                    return null;
                }
            };
        }
    }
    return selector;
}

Example 12 with ObjectColumnSelector

use of io.druid.segment.ObjectColumnSelector in project druid by druid-io.

the class DumpSegment method runDump.

private void runDump(final Injector injector, final QueryableIndex index) throws IOException {
    final ObjectMapper objectMapper = injector.getInstance(Key.get(ObjectMapper.class, Json.class));
    final QueryableIndexStorageAdapter adapter = new QueryableIndexStorageAdapter(index);
    final List<String> columnNames = getColumnsToInclude(index);
    final DimFilter filter = filterJson != null ? objectMapper.readValue(filterJson, DimFilter.class) : null;
    final Sequence<Cursor> cursors = adapter.makeCursors(Filters.toFilter(filter), index.getDataInterval().withChronology(ISOChronology.getInstanceUTC()), VirtualColumns.EMPTY, Granularities.ALL, false);
    withOutputStream(new Function<OutputStream, Object>() {

        @Override
        public Object apply(final OutputStream out) {
            final Sequence<Object> sequence = Sequences.map(cursors, new Function<Cursor, Object>() {

                @Override
                public Object apply(Cursor cursor) {
                    final List<ObjectColumnSelector> selectors = Lists.newArrayList();
                    for (String columnName : columnNames) {
                        selectors.add(makeSelector(columnName, index.getColumn(columnName), cursor));
                    }
                    while (!cursor.isDone()) {
                        final Map<String, Object> row = Maps.newLinkedHashMap();
                        for (int i = 0; i < columnNames.size(); i++) {
                            final String columnName = columnNames.get(i);
                            final Object value = selectors.get(i).get();
                            if (timeISO8601 && columnNames.get(i).equals(Column.TIME_COLUMN_NAME)) {
                                row.put(columnName, new DateTime(value, DateTimeZone.UTC).toString());
                            } else {
                                row.put(columnName, value);
                            }
                        }
                        try {
                            out.write(objectMapper.writeValueAsBytes(row));
                            out.write('\n');
                        } catch (IOException e) {
                            throw Throwables.propagate(e);
                        }
                        cursor.advance();
                    }
                    return null;
                }
            });
            evaluateSequenceForSideEffects(sequence);
            return null;
        }
    });
}

Example 13 with ObjectColumnSelector

use of io.druid.segment.ObjectColumnSelector in project druid by druid-io.

the class ScanQueryEngine method process.

public Sequence<ScanResultValue> process(final ScanQuery query, final Segment segment, final Map<String, Object> responseContext) {
    if (responseContext.get(ScanQueryRunnerFactory.CTX_COUNT) != null) {
        int count = (int) responseContext.get(ScanQueryRunnerFactory.CTX_COUNT);
        if (count >= query.getLimit()) {
            return Sequences.empty();
        }
    }
    final Long timeoutAt = (long) responseContext.get(ScanQueryRunnerFactory.CTX_TIMEOUT_AT);
    final long start = System.currentTimeMillis();
    final StorageAdapter adapter = segment.asStorageAdapter();
    if (adapter == null) {
        throw new ISE("Null storage adapter found. Probably trying to issue a query against a segment being memory unmapped.");
    }
    List<String> allDims = Lists.newLinkedList(adapter.getAvailableDimensions());
    List<String> allMetrics = Lists.newLinkedList(adapter.getAvailableMetrics());
    final List<String> allColumns = Lists.newLinkedList();
    if (query.getColumns() != null && !query.getColumns().isEmpty()) {
        if (!query.getColumns().contains(ScanResultValue.timestampKey)) {
            allColumns.add(ScanResultValue.timestampKey);
        }
        allColumns.addAll(query.getColumns());
        allDims.retainAll(query.getColumns());
        allMetrics.retainAll(query.getColumns());
    } else {
        if (!allDims.contains(ScanResultValue.timestampKey)) {
            allColumns.add(ScanResultValue.timestampKey);
        }
        allColumns.addAll(allDims);
        allColumns.addAll(allMetrics);
    }
    final List<DimensionSpec> dims = DefaultDimensionSpec.toSpec(allDims);
    final List<String> metrics = allMetrics;
    final List<Interval> intervals = query.getQuerySegmentSpec().getIntervals();
    Preconditions.checkArgument(intervals.size() == 1, "Can only handle a single interval, got[%s]", intervals);
    final String segmentId = segment.getIdentifier();
    final Filter filter = Filters.convertToCNFFromQueryContext(query, Filters.toFilter(query.getDimensionsFilter()));
    if (responseContext.get(ScanQueryRunnerFactory.CTX_COUNT) == null) {
        responseContext.put(ScanQueryRunnerFactory.CTX_COUNT, 0);
    }
    final int limit = query.getLimit() - (int) responseContext.get(ScanQueryRunnerFactory.CTX_COUNT);
    return Sequences.concat(Sequences.map(adapter.makeCursors(filter, intervals.get(0), VirtualColumns.EMPTY, Granularities.ALL, query.isDescending()), new Function<Cursor, Sequence<ScanResultValue>>() {

        @Override
        public Sequence<ScanResultValue> apply(final Cursor cursor) {
            return new BaseSequence<>(new BaseSequence.IteratorMaker<ScanResultValue, Iterator<ScanResultValue>>() {

                @Override
                public Iterator<ScanResultValue> make() {
                    final LongColumnSelector timestampColumnSelector = cursor.makeLongColumnSelector(Column.TIME_COLUMN_NAME);
                    final List<ColumnSelectorPlus<SelectQueryEngine.SelectColumnSelectorStrategy>> selectorPlusList = Arrays.asList(DimensionHandlerUtils.createColumnSelectorPluses(STRATEGY_FACTORY, Lists.newArrayList(dims), cursor));
                    final Map<String, ObjectColumnSelector> metSelectors = Maps.newHashMap();
                    for (String metric : metrics) {
                        final ObjectColumnSelector metricSelector = cursor.makeObjectColumnSelector(metric);
                        metSelectors.put(metric, metricSelector);
                    }
                    final int batchSize = query.getBatchSize();
                    return new Iterator<ScanResultValue>() {

                        private int offset = 0;

                        @Override
                        public boolean hasNext() {
                            return !cursor.isDone() && offset < limit;
                        }

                        @Override
                        public ScanResultValue next() {
                            if (System.currentTimeMillis() >= timeoutAt) {
                                throw new QueryInterruptedException(new TimeoutException());
                            }
                            int lastOffset = offset;
                            Object events = null;
                            String resultFormat = query.getResultFormat();
                            if (ScanQuery.RESULT_FORMAT_VALUE_VECTOR.equals(resultFormat)) {
                                throw new UnsupportedOperationException("valueVector is not supported now");
                            } else if (ScanQuery.RESULT_FORMAT_COMPACTED_LIST.equals(resultFormat)) {
                                events = rowsToCompactedList();
                            } else {
                                events = rowsToList();
                            }
                            responseContext.put(ScanQueryRunnerFactory.CTX_COUNT, (int) responseContext.get(ScanQueryRunnerFactory.CTX_COUNT) + (offset - lastOffset));
                            responseContext.put(ScanQueryRunnerFactory.CTX_TIMEOUT_AT, timeoutAt - (System.currentTimeMillis() - start));
                            return new ScanResultValue(segmentId, allColumns, events);
                        }

                        @Override
                        public void remove() {
                            throw new UnsupportedOperationException();
                        }

                        private Object rowsToCompactedList() {
                            return Lists.transform((List<Map<String, Object>>) rowsToList(), new Function<Map<String, Object>, Object>() {

                                @Override
                                public Object apply(Map<String, Object> input) {
                                    List eventValues = Lists.newArrayListWithExpectedSize(allColumns.size());
                                    for (String expectedColumn : allColumns) {
                                        eventValues.add(input.get(expectedColumn));
                                    }
                                    return eventValues;
                                }
                            });
                        }

                        private Object rowsToList() {
                            List<Map<String, Object>> events = Lists.newArrayListWithCapacity(batchSize);
                            for (int i = 0; !cursor.isDone() && i < batchSize && offset < limit; cursor.advance(), i++, offset++) {
                                final Map<String, Object> theEvent = SelectQueryEngine.singleEvent(ScanResultValue.timestampKey, timestampColumnSelector, selectorPlusList, metSelectors);
                                events.add(theEvent);
                            }
                            return events;
                        }

                        private Object rowsToValueVector() {
                            // only support list now, we can support ValueVector or Arrow in future
                            return rowsToList();
                        }
                    };
                }

                @Override
                public void cleanup(Iterator<ScanResultValue> iterFromMake) {
                }
            });
        }
    }));
}

Example 14 with ObjectColumnSelector

use of io.druid.segment.ObjectColumnSelector in project druid by druid-io.

the class ExpressionVirtualColumnTest method testObjectSelector.

@Test
public void testObjectSelector() {
    final ObjectColumnSelector selector = XPLUSY.makeObjectColumnSelector("expr", COLUMN_SELECTOR_FACTORY);
    COLUMN_SELECTOR_FACTORY.setRow(ROW0);
    Assert.assertEquals(null, selector.get());
    COLUMN_SELECTOR_FACTORY.setRow(ROW1);
    Assert.assertEquals(null, selector.get());
    COLUMN_SELECTOR_FACTORY.setRow(ROW2);
    Assert.assertEquals(5.1d, selector.get());
}

Example 15 with ObjectColumnSelector

use of io.druid.segment.ObjectColumnSelector in project druid by druid-io.

the class VirtualColumnsTest method testMakeSelectors.

@Test
public void testMakeSelectors() {
    final VirtualColumns virtualColumns = makeVirtualColumns();
    final ObjectColumnSelector objectSelector = virtualColumns.makeObjectColumnSelector("expr", null);
    final DimensionSelector dimensionSelector = virtualColumns.makeDimensionSelector(new DefaultDimensionSpec("expr", "x"), null);
    final DimensionSelector extractionDimensionSelector = virtualColumns.makeDimensionSelector(new ExtractionDimensionSpec("expr", "x", new BucketExtractionFn(1.0, 0.5)), null);
    final FloatColumnSelector floatSelector = virtualColumns.makeFloatColumnSelector("expr", null);
    final LongColumnSelector longSelector = virtualColumns.makeLongColumnSelector("expr", null);
    Assert.assertEquals(1L, objectSelector.get());
    Assert.assertEquals("1", dimensionSelector.lookupName(dimensionSelector.getRow().get(0)));
    Assert.assertEquals("0.5", extractionDimensionSelector.lookupName(extractionDimensionSelector.getRow().get(0)));
    Assert.assertEquals(1.0f, floatSelector.get(), 0.0f);
    Assert.assertEquals(1L, longSelector.get());
}