Examples with StorageAdapter org.apache.druid.segment.StorageAdapter used on opensource projects

Example 6 with StorageAdapter

use of org.apache.druid.segment.StorageAdapter in project druid by druid-io.

the class HashJoinSegmentTest method setUp.

@Before
public void setUp() throws IOException {
    allReferencesAcquireCount = 0;
    allReferencesCloseCount = 0;
    referencedSegmentAcquireCount = 0;
    referencedSegmentClosedCount = 0;
    indexedTableJoinableReferenceAcquireCount = 0;
    indexedTableJoinableReferenceCloseCount = 0;
    j0Closed = false;
    j1Closed = false;
    baseSegment = new QueryableIndexSegment(JoinTestHelper.createFactIndexBuilder(temporaryFolder.newFolder()).buildMMappedIndex(), SegmentId.dummy("facts"));
    List<JoinableClause> joinableClauses = ImmutableList.of(new JoinableClause("j0.", new IndexedTableJoinable(JoinTestHelper.createCountriesIndexedTable()) {

        @Override
        public Optional<Closeable> acquireReferences() {
            if (!j0Closed) {
                indexedTableJoinableReferenceAcquireCount++;
                Closer closer = Closer.create();
                closer.register(() -> indexedTableJoinableReferenceCloseCount++);
                return Optional.of(closer);
            }
            return Optional.empty();
        }
    }, JoinType.LEFT, JoinConditionAnalysis.forExpression("1", "j0.", ExprMacroTable.nil())), new JoinableClause("j1.", new IndexedTableJoinable(JoinTestHelper.createRegionsIndexedTable()) {

        @Override
        public Optional<Closeable> acquireReferences() {
            if (!j1Closed) {
                indexedTableJoinableReferenceAcquireCount++;
                Closer closer = Closer.create();
                closer.register(() -> indexedTableJoinableReferenceCloseCount++);
                return Optional.of(closer);
            }
            return Optional.empty();
        }
    }, JoinType.LEFT, JoinConditionAnalysis.forExpression("1", "j1.", ExprMacroTable.nil())));
    referencedSegment = ReferenceCountingSegment.wrapRootGenerationSegment(baseSegment);
    SegmentReference testWrapper = new SegmentReference() {

        @Override
        public Optional<Closeable> acquireReferences() {
            Closer closer = Closer.create();
            return referencedSegment.acquireReferences().map(closeable -> {
                referencedSegmentAcquireCount++;
                closer.register(closeable);
                closer.register(() -> referencedSegmentClosedCount++);
                return closer;
            });
        }

        @Override
        public SegmentId getId() {
            return referencedSegment.getId();
        }

        @Override
        public Interval getDataInterval() {
            return referencedSegment.getDataInterval();
        }

        @Nullable
        @Override
        public QueryableIndex asQueryableIndex() {
            return referencedSegment.asQueryableIndex();
        }

        @Override
        public StorageAdapter asStorageAdapter() {
            return referencedSegment.asStorageAdapter();
        }

        @Override
        public void close() {
            referencedSegment.close();
        }
    };
    hashJoinSegment = new HashJoinSegment(testWrapper, null, joinableClauses, null) {

        @Override
        public Optional<Closeable> acquireReferences() {
            Closer closer = Closer.create();
            return super.acquireReferences().map(closeable -> {
                allReferencesAcquireCount++;
                closer.register(closeable);
                closer.register(() -> allReferencesCloseCount++);
                return closer;
            });
        }
    };
}

Example 7 with StorageAdapter

use of org.apache.druid.segment.StorageAdapter in project druid by druid-io.

the class ExpressionSelectorsTest method test_double_bindings.

@Test
public void test_double_bindings() {
    final String columnName = "double3";
    for (StorageAdapter adapter : ADAPTERS) {
        Sequence<Cursor> cursorSequence = adapter.makeCursors(null, adapter.getInterval(), VirtualColumns.EMPTY, Granularities.ALL, false, null);
        List<Cursor> flatten = cursorSequence.toList();
        for (Cursor cursor : flatten) {
            ColumnSelectorFactory factory = cursor.getColumnSelectorFactory();
            // an assortment of plans
            ExpressionPlan plan = ExpressionPlanner.plan(adapter, Parser.parse("\"double3\"", TestExprMacroTable.INSTANCE));
            ExpressionPlan plan2 = ExpressionPlanner.plan(adapter, Parser.parse("\"double3\" + 3.0", TestExprMacroTable.INSTANCE));
            Expr.ObjectBinding bindings = ExpressionSelectors.createBindings(factory, plan);
            Expr.ObjectBinding bindings2 = ExpressionSelectors.createBindings(factory, plan2);
            ColumnValueSelector valueSelector = factory.makeColumnValueSelector(columnName);
            while (!cursor.isDone()) {
                Object bindingVal = bindings.get(columnName);
                Object bindingVal2 = bindings2.get(columnName);
                if (valueSelector.isNull()) {
                    Assert.assertNull(valueSelector.getObject());
                    Assert.assertNull(bindingVal);
                    Assert.assertNull(bindingVal2);
                } else {
                    Assert.assertEquals(valueSelector.getObject(), bindingVal);
                    Assert.assertEquals(valueSelector.getDouble(), bindingVal);
                    Assert.assertEquals(valueSelector.getObject(), bindingVal2);
                    Assert.assertEquals(valueSelector.getDouble(), bindingVal2);
                }
                cursor.advance();
            }
        }
    }
}

Example 8 with StorageAdapter

use of org.apache.druid.segment.StorageAdapter in project druid by druid-io.

the class ExpressionSelectorsTest method test_multi_value_string_bindings.

@Test
public void test_multi_value_string_bindings() {
    final String columnName = "multi-string3";
    for (StorageAdapter adapter : ADAPTERS) {
        Sequence<Cursor> cursorSequence = adapter.makeCursors(null, adapter.getInterval(), VirtualColumns.EMPTY, Granularities.ALL, false, null);
        List<Cursor> flatten = cursorSequence.toList();
        for (Cursor cursor : flatten) {
            ColumnSelectorFactory factory = cursor.getColumnSelectorFactory();
            // identifier, uses dimension selector supplier supplier, no null coercion
            ExpressionPlan plan = ExpressionPlanner.plan(adapter, Parser.parse("\"multi-string3\"", TestExprMacroTable.INSTANCE));
            // array output, uses object selector supplier, no null coercion
            ExpressionPlan plan2 = ExpressionPlanner.plan(adapter, Parser.parse("array_append(\"multi-string3\", 'foo')", TestExprMacroTable.INSTANCE));
            // array input, uses dimension selector supplier, no null coercion
            ExpressionPlan plan3 = ExpressionPlanner.plan(adapter, Parser.parse("array_length(\"multi-string3\")", TestExprMacroTable.INSTANCE));
            // used as scalar, has null coercion
            ExpressionPlan plan4 = ExpressionPlanner.plan(adapter, Parser.parse("concat(\"multi-string3\", 'foo')", TestExprMacroTable.INSTANCE));
            Expr.ObjectBinding bindings = ExpressionSelectors.createBindings(factory, plan);
            Expr.ObjectBinding bindings2 = ExpressionSelectors.createBindings(factory, plan2);
            Expr.ObjectBinding bindings3 = ExpressionSelectors.createBindings(factory, plan3);
            Expr.ObjectBinding bindings4 = ExpressionSelectors.createBindings(factory, plan4);
            DimensionSelector dimSelector = factory.makeDimensionSelector(DefaultDimensionSpec.of(columnName));
            ColumnValueSelector valueSelector = factory.makeColumnValueSelector(columnName);
            while (!cursor.isDone()) {
                Object dimSelectorVal = dimSelector.getObject();
                Object valueSelectorVal = valueSelector.getObject();
                Object bindingVal = bindings.get(columnName);
                Object bindingVal2 = bindings2.get(columnName);
                Object bindingVal3 = bindings3.get(columnName);
                Object bindingVal4 = bindings4.get(columnName);
                if (dimSelectorVal == null) {
                    Assert.assertNull(dimSelectorVal);
                    Assert.assertNull(valueSelectorVal);
                    Assert.assertNull(bindingVal);
                    Assert.assertNull(bindingVal2);
                    Assert.assertNull(bindingVal3);
                    // binding4 has null coercion
                    Assert.assertArrayEquals(new Object[] { null }, (Object[]) bindingVal4);
                } else {
                    Assert.assertArrayEquals(((List) dimSelectorVal).toArray(), (Object[]) bindingVal);
                    Assert.assertArrayEquals(((List) valueSelectorVal).toArray(), (Object[]) bindingVal);
                    Assert.assertArrayEquals(((List) dimSelectorVal).toArray(), (Object[]) bindingVal2);
                    Assert.assertArrayEquals(((List) valueSelectorVal).toArray(), (Object[]) bindingVal2);
                    Assert.assertArrayEquals(((List) dimSelectorVal).toArray(), (Object[]) bindingVal3);
                    Assert.assertArrayEquals(((List) valueSelectorVal).toArray(), (Object[]) bindingVal3);
                }
                cursor.advance();
            }
        }
    }
}

Example 9 with StorageAdapter

use of org.apache.druid.segment.StorageAdapter in project druid by druid-io.

the class ExpressionSelectorsTest method test_single_value_string_bindings.

@Test
public void test_single_value_string_bindings() {
    final String columnName = "string3";
    for (StorageAdapter adapter : ADAPTERS) {
        Sequence<Cursor> cursorSequence = adapter.makeCursors(null, adapter.getInterval(), VirtualColumns.EMPTY, Granularities.ALL, false, null);
        List<Cursor> flatten = cursorSequence.toList();
        for (Cursor cursor : flatten) {
            ColumnSelectorFactory factory = cursor.getColumnSelectorFactory();
            ExpressionPlan plan = ExpressionPlanner.plan(adapter, Parser.parse("\"string3\"", TestExprMacroTable.INSTANCE));
            ExpressionPlan plan2 = ExpressionPlanner.plan(adapter, Parser.parse("concat(\"string3\", 'foo')", TestExprMacroTable.INSTANCE));
            Expr.ObjectBinding bindings = ExpressionSelectors.createBindings(factory, plan);
            Expr.ObjectBinding bindings2 = ExpressionSelectors.createBindings(factory, plan2);
            DimensionSelector dimSelector = factory.makeDimensionSelector(DefaultDimensionSpec.of(columnName));
            ColumnValueSelector valueSelector = factory.makeColumnValueSelector(columnName);
            // realtime index needs to handle as multi-value in case any new values are added during processing
            final boolean isMultiVal = factory.getColumnCapabilities(columnName) == null || factory.getColumnCapabilities(columnName).hasMultipleValues().isMaybeTrue();
            while (!cursor.isDone()) {
                Object dimSelectorVal = dimSelector.getObject();
                Object valueSelectorVal = valueSelector.getObject();
                Object bindingVal = bindings.get(columnName);
                Object bindingVal2 = bindings2.get(columnName);
                if (dimSelectorVal == null) {
                    Assert.assertNull(dimSelectorVal);
                    Assert.assertNull(valueSelectorVal);
                    Assert.assertNull(bindingVal);
                    if (isMultiVal) {
                        Assert.assertNull(((Object[]) bindingVal2)[0]);
                    } else {
                        Assert.assertNull(bindingVal2);
                    }
                } else {
                    if (isMultiVal) {
                        Assert.assertEquals(dimSelectorVal, ((Object[]) bindingVal)[0]);
                        Assert.assertEquals(valueSelectorVal, ((Object[]) bindingVal)[0]);
                        Assert.assertEquals(dimSelectorVal, ((Object[]) bindingVal2)[0]);
                        Assert.assertEquals(valueSelectorVal, ((Object[]) bindingVal2)[0]);
                    } else {
                        Assert.assertEquals(dimSelectorVal, bindingVal);
                        Assert.assertEquals(valueSelectorVal, bindingVal);
                        Assert.assertEquals(dimSelectorVal, bindingVal2);
                        Assert.assertEquals(valueSelectorVal, bindingVal2);
                    }
                }
                cursor.advance();
            }
        }
    }
}

Example 10 with StorageAdapter

use of org.apache.druid.segment.StorageAdapter in project druid by druid-io.

the class TopNQueryEngine method query.

/**
 * Do the thing - process a {@link StorageAdapter} into a {@link Sequence} of {@link TopNResultValue}, with one of the
 * fine {@link TopNAlgorithm} available chosen based on the type of column being aggregated. The algorithm provides a
 * mapping function to process rows from the adapter {@link org.apache.druid.segment.Cursor} to apply
 * {@link AggregatorFactory} and create or update {@link TopNResultValue}
 */
public Sequence<Result<TopNResultValue>> query(final TopNQuery query, final StorageAdapter adapter, @Nullable final TopNQueryMetrics queryMetrics) {
    if (adapter == null) {
        throw new SegmentMissingException("Null storage adapter found. Probably trying to issue a query against a segment being memory unmapped.");
    }
    final List<Interval> queryIntervals = query.getQuerySegmentSpec().getIntervals();
    final Filter filter = Filters.convertToCNFFromQueryContext(query, Filters.toFilter(query.getDimensionsFilter()));
    final Granularity granularity = query.getGranularity();
    final TopNMapFn mapFn = getMapFn(query, adapter, queryMetrics);
    Preconditions.checkArgument(queryIntervals.size() == 1, "Can only handle a single interval, got[%s]", queryIntervals);
    return Sequences.filter(Sequences.map(adapter.makeCursors(filter, queryIntervals.get(0), query.getVirtualColumns(), granularity, query.isDescending(), queryMetrics), input -> {
        if (queryMetrics != null) {
            queryMetrics.cursor(input);
        }
        return mapFn.apply(input, queryMetrics);
    }), Predicates.notNull());
}