Examples with VirtualColumns org.apache.druid.segment.VirtualColumns used on opensource projects

Example 11 with VirtualColumns

use of org.apache.druid.segment.VirtualColumns in project druid by druid-io.

the class VirtualColumnsTest method testSerde.

@Test
public void testSerde() throws Exception {
    final ObjectMapper mapper = TestHelper.makeJsonMapper();
    final ImmutableList<VirtualColumn> theColumns = ImmutableList.of(new ExpressionVirtualColumn("expr", "x + y", ColumnType.FLOAT, TestExprMacroTable.INSTANCE), new ExpressionVirtualColumn("expr2", "x + z", ColumnType.FLOAT, TestExprMacroTable.INSTANCE));
    final VirtualColumns virtualColumns = VirtualColumns.create(theColumns);
    Assert.assertEquals(virtualColumns, mapper.readValue(mapper.writeValueAsString(virtualColumns), VirtualColumns.class));
    Assert.assertEquals(theColumns, mapper.readValue(mapper.writeValueAsString(virtualColumns), mapper.getTypeFactory().constructParametricType(List.class, VirtualColumn.class)));
}

Example 12 with VirtualColumns

use of org.apache.druid.segment.VirtualColumns in project druid by druid-io.

the class HashJoinSegmentStorageAdapterTest method test_makeCursors_factToCountryUsingVirtualColumnUsingLookup.

@Test
public void test_makeCursors_factToCountryUsingVirtualColumnUsingLookup() {
    List<JoinableClause> joinableClauses = ImmutableList.of(new JoinableClause(FACT_TO_COUNTRY_ON_ISO_CODE_PREFIX, LookupJoinable.wrap(countryIsoCodeToNameLookup), JoinType.INNER, JoinConditionAnalysis.forExpression(StringUtils.format("\"%sk\" == virtual", FACT_TO_COUNTRY_ON_ISO_CODE_PREFIX), FACT_TO_COUNTRY_ON_ISO_CODE_PREFIX, ExprMacroTable.nil())));
    VirtualColumns virtualColumns = VirtualColumns.create(Collections.singletonList(makeExpressionVirtualColumn("concat(substring(countryIsoCode, 0, 1),'L')")));
    JoinFilterPreAnalysis joinFilterPreAnalysis = makeDefaultConfigPreAnalysis(null, joinableClauses, virtualColumns);
    JoinTestHelper.verifyCursors(new HashJoinSegmentStorageAdapter(factSegment.asStorageAdapter(), joinableClauses, joinFilterPreAnalysis).makeCursors(null, Intervals.ETERNITY, virtualColumns, Granularities.ALL, false, null), ImmutableList.of("page", "countryIsoCode", "virtual", FACT_TO_COUNTRY_ON_ISO_CODE_PREFIX + "k", FACT_TO_COUNTRY_ON_ISO_CODE_PREFIX + "v"), ImmutableList.of(new Object[] { "Golpe de Estado en Chile de 1973", "CL", "CL", "CL", "Chile" }, new Object[] { "Didier Leclair", "CA", "CL", "CL", "Chile" }, new Object[] { "Les Argonautes", "CA", "CL", "CL", "Chile" }, new Object[] { "Sarah Michelle Gellar", "CA", "CL", "CL", "Chile" }));
}

Example 13 with VirtualColumns

use of org.apache.druid.segment.VirtualColumns in project druid by druid-io.

the class ExpressionVectorSelectorBenchmark method scan.

@Benchmark
@BenchmarkMode(Mode.AverageTime)
@OutputTimeUnit(TimeUnit.MILLISECONDS)
public void scan(Blackhole blackhole) {
    final VirtualColumns virtualColumns = VirtualColumns.create(ImmutableList.of(new ExpressionVirtualColumn("v", expression, ExpressionType.toColumnType(outputType), TestExprMacroTable.INSTANCE)));
    if (vectorize) {
        VectorCursor cursor = new QueryableIndexStorageAdapter(index).makeVectorCursor(null, index.getDataInterval(), virtualColumns, false, 512, null);
        if (outputType.isNumeric()) {
            VectorValueSelector selector = cursor.getColumnSelectorFactory().makeValueSelector("v");
            if (outputType.is(ExprType.DOUBLE)) {
                while (!cursor.isDone()) {
                    blackhole.consume(selector.getDoubleVector());
                    blackhole.consume(selector.getNullVector());
                    cursor.advance();
                }
            } else {
                while (!cursor.isDone()) {
                    blackhole.consume(selector.getLongVector());
                    blackhole.consume(selector.getNullVector());
                    cursor.advance();
                }
            }
            closer.register(cursor);
        }
    } else {
        Sequence<Cursor> cursors = new QueryableIndexStorageAdapter(index).makeCursors(null, index.getDataInterval(), virtualColumns, Granularities.ALL, false, null);
        int rowCount = cursors.map(cursor -> {
            final ColumnValueSelector selector = cursor.getColumnSelectorFactory().makeColumnValueSelector("v");
            int rows = 0;
            while (!cursor.isDone()) {
                blackhole.consume(selector.getObject());
                rows++;
                cursor.advance();
            }
            return rows;
        }).accumulate(0, (acc, in) -> acc + in);
        blackhole.consume(rowCount);
    }
}

Example 14 with VirtualColumns

use of org.apache.druid.segment.VirtualColumns in project druid by druid-io.

the class HashJoinSegmentStorageAdapterTest method test_makeCursors_factToCountryUsingVirtualColumn.

@Test
public void test_makeCursors_factToCountryUsingVirtualColumn() {
    List<JoinableClause> joinableClauses = ImmutableList.of(new JoinableClause(FACT_TO_COUNTRY_ON_ISO_CODE_PREFIX, new IndexedTableJoinable(countriesTable), JoinType.INNER, JoinConditionAnalysis.forExpression(StringUtils.format("\"%scountryIsoCode\" == virtual", FACT_TO_COUNTRY_ON_ISO_CODE_PREFIX), FACT_TO_COUNTRY_ON_ISO_CODE_PREFIX, ExprMacroTable.nil())));
    VirtualColumns virtualColumns = VirtualColumns.create(Collections.singletonList(makeExpressionVirtualColumn("concat(substring(countryIsoCode, 0, 1),'L')")));
    JoinFilterPreAnalysis joinFilterPreAnalysis = makeDefaultConfigPreAnalysis(null, joinableClauses, virtualColumns);
    JoinTestHelper.verifyCursors(new HashJoinSegmentStorageAdapter(factSegment.asStorageAdapter(), joinableClauses, joinFilterPreAnalysis).makeCursors(null, Intervals.ETERNITY, virtualColumns, Granularities.ALL, false, null), ImmutableList.of("page", "countryIsoCode", "virtual", FACT_TO_COUNTRY_ON_ISO_CODE_PREFIX + "countryIsoCode", FACT_TO_COUNTRY_ON_ISO_CODE_PREFIX + "countryName"), ImmutableList.of(new Object[] { "Golpe de Estado en Chile de 1973", "CL", "CL", "CL", "Chile" }, new Object[] { "Didier Leclair", "CA", "CL", "CL", "Chile" }, new Object[] { "Les Argonautes", "CA", "CL", "CL", "Chile" }, new Object[] { "Sarah Michelle Gellar", "CA", "CL", "CL", "Chile" }));
}

Example 15 with VirtualColumns

use of org.apache.druid.segment.VirtualColumns in project druid by druid-io.

the class HashJoinSegmentStorageAdapter method makeCursors.

@Override
public Sequence<Cursor> makeCursors(@Nullable final Filter filter, @Nonnull final Interval interval, @Nonnull final VirtualColumns virtualColumns, @Nonnull final Granularity gran, final boolean descending, @Nullable final QueryMetrics<?> queryMetrics) {
    final Filter combinedFilter = baseFilterAnd(filter);
    if (clauses.isEmpty()) {
        return baseAdapter.makeCursors(combinedFilter, interval, virtualColumns, gran, descending, queryMetrics);
    }
    // Filter pre-analysis key implied by the call to "makeCursors". We need to sanity-check that it matches
    // the actual pre-analysis that was done. Note: we can't infer a rewrite config from the "makeCursors" call (it
    // requires access to the query context) so we'll need to skip sanity-checking it, by re-using the one present
    // in the cached key.)
    final JoinFilterPreAnalysisKey keyIn = new JoinFilterPreAnalysisKey(joinFilterPreAnalysis.getKey().getRewriteConfig(), clauses, virtualColumns, combinedFilter);
    final JoinFilterPreAnalysisKey keyCached = joinFilterPreAnalysis.getKey();
    if (!keyIn.equals(keyCached)) {
        // It is a bug if this happens. The implied key and the cached key should always match.
        throw new ISE("Pre-analysis mismatch, cannot execute query");
    }
    final List<VirtualColumn> preJoinVirtualColumns = new ArrayList<>();
    final List<VirtualColumn> postJoinVirtualColumns = new ArrayList<>();
    determineBaseColumnsWithPreAndPostJoinVirtualColumns(virtualColumns, preJoinVirtualColumns, postJoinVirtualColumns);
    // We merge the filter on base table specified by the user and filter on the base table that is pushed from
    // the join
    JoinFilterSplit joinFilterSplit = JoinFilterAnalyzer.splitFilter(joinFilterPreAnalysis, baseFilter);
    preJoinVirtualColumns.addAll(joinFilterSplit.getPushDownVirtualColumns());
    final Sequence<Cursor> baseCursorSequence = baseAdapter.makeCursors(joinFilterSplit.getBaseTableFilter().isPresent() ? joinFilterSplit.getBaseTableFilter().get() : null, interval, VirtualColumns.create(preJoinVirtualColumns), gran, descending, queryMetrics);
    Closer joinablesCloser = Closer.create();
    return Sequences.<Cursor, Cursor>map(baseCursorSequence, cursor -> {
        assert cursor != null;
        Cursor retVal = cursor;
        for (JoinableClause clause : clauses) {
            retVal = HashJoinEngine.makeJoinCursor(retVal, clause, descending, joinablesCloser);
        }
        return PostJoinCursor.wrap(retVal, VirtualColumns.create(postJoinVirtualColumns), joinFilterSplit.getJoinTableFilter().orElse(null));
    }).withBaggage(joinablesCloser);
}