Search in sources :

Example 26 with Sequence

use of org.apache.druid.java.util.common.guava.Sequence in project druid by druid-io.

the class SqlResourceTest method setUp.

@Before
public void setUp() throws Exception {
    final QueryScheduler scheduler = new QueryScheduler(5, ManualQueryPrioritizationStrategy.INSTANCE, new HiLoQueryLaningStrategy(40), new ServerConfig()) {

        @Override
        public <T> Sequence<T> run(Query<?> query, Sequence<T> resultSequence) {
            return super.run(query, new LazySequence<T>(() -> {
                if (sleep) {
                    try {
                        // pretend to be a query that is waiting on results
                        Thread.sleep(500);
                    } catch (InterruptedException ignored) {
                    }
                }
                return resultSequence;
            }));
        }
    };
    executorService = MoreExecutors.listeningDecorator(Execs.multiThreaded(8, "test_sql_resource_%s"));
    walker = CalciteTests.createMockWalker(conglomerate, temporaryFolder.newFolder(), scheduler);
    final PlannerConfig plannerConfig = new PlannerConfig() {

        @Override
        public boolean shouldSerializeComplexValues() {
            return false;
        }
    };
    final DruidSchemaCatalog rootSchema = CalciteTests.createMockRootSchema(conglomerate, walker, plannerConfig, CalciteTests.TEST_AUTHORIZER_MAPPER);
    final DruidOperatorTable operatorTable = CalciteTests.createOperatorTable();
    final ExprMacroTable macroTable = CalciteTests.createExprMacroTable();
    req = EasyMock.createStrictMock(HttpServletRequest.class);
    EasyMock.expect(req.getRemoteAddr()).andReturn(null).once();
    EasyMock.expect(req.getAttribute(AuthConfig.DRUID_AUTHENTICATION_RESULT)).andReturn(CalciteTests.REGULAR_USER_AUTH_RESULT).anyTimes();
    EasyMock.expect(req.getAttribute(AuthConfig.DRUID_ALLOW_UNSECURED_PATH)).andReturn(null).anyTimes();
    EasyMock.expect(req.getAttribute(AuthConfig.DRUID_AUTHORIZATION_CHECKED)).andReturn(null).anyTimes();
    EasyMock.expect(req.getAttribute(AuthConfig.DRUID_AUTHENTICATION_RESULT)).andReturn(CalciteTests.REGULAR_USER_AUTH_RESULT).anyTimes();
    req.setAttribute(AuthConfig.DRUID_AUTHORIZATION_CHECKED, true);
    EasyMock.expectLastCall().anyTimes();
    EasyMock.expect(req.getAttribute(AuthConfig.DRUID_AUTHENTICATION_RESULT)).andReturn(CalciteTests.REGULAR_USER_AUTH_RESULT).anyTimes();
    EasyMock.replay(req);
    testRequestLogger = new TestRequestLogger();
    final PlannerFactory plannerFactory = new PlannerFactory(rootSchema, CalciteTests.createMockQueryMakerFactory(walker, conglomerate), operatorTable, macroTable, plannerConfig, CalciteTests.TEST_AUTHORIZER_MAPPER, CalciteTests.getJsonMapper(), CalciteTests.DRUID_SCHEMA_NAME);
    lifecycleManager = new SqlLifecycleManager() {

        @Override
        public void add(String sqlQueryId, SqlLifecycle lifecycle) {
            super.add(sqlQueryId, lifecycle);
            if (lifecycleAddLatch != null) {
                lifecycleAddLatch.countDown();
            }
        }
    };
    final ServiceEmitter emitter = new NoopServiceEmitter();
    sqlLifecycleFactory = new SqlLifecycleFactory(plannerFactory, emitter, testRequestLogger, scheduler) {

        @Override
        public SqlLifecycle factorize() {
            return new TestSqlLifecycle(plannerFactory, emitter, testRequestLogger, scheduler, System.currentTimeMillis(), System.nanoTime(), validateAndAuthorizeLatchSupplier, planLatchSupplier, executeLatchSupplier, sequenceMapFnSupplier);
        }
    };
    resource = new SqlResource(JSON_MAPPER, CalciteTests.TEST_AUTHORIZER_MAPPER, sqlLifecycleFactory, lifecycleManager, new ServerConfig());
}
Also used : SqlLifecycleManager(org.apache.druid.sql.SqlLifecycleManager) ServiceEmitter(org.apache.druid.java.util.emitter.service.ServiceEmitter) NoopServiceEmitter(org.apache.druid.server.metrics.NoopServiceEmitter) QueryScheduler(org.apache.druid.server.QueryScheduler) BaseQuery(org.apache.druid.query.BaseQuery) Query(org.apache.druid.query.Query) HiLoQueryLaningStrategy(org.apache.druid.server.scheduling.HiLoQueryLaningStrategy) SqlLifecycle(org.apache.druid.sql.SqlLifecycle) NoopServiceEmitter(org.apache.druid.server.metrics.NoopServiceEmitter) Sequence(org.apache.druid.java.util.common.guava.Sequence) LazySequence(org.apache.druid.java.util.common.guava.LazySequence) QueryInterruptedException(org.apache.druid.query.QueryInterruptedException) DruidOperatorTable(org.apache.druid.sql.calcite.planner.DruidOperatorTable) ExprMacroTable(org.apache.druid.math.expr.ExprMacroTable) TestRequestLogger(org.apache.druid.server.log.TestRequestLogger) HttpServletRequest(javax.servlet.http.HttpServletRequest) ServerConfig(org.apache.druid.server.initialization.ServerConfig) PlannerConfig(org.apache.druid.sql.calcite.planner.PlannerConfig) DruidSchemaCatalog(org.apache.druid.sql.calcite.schema.DruidSchemaCatalog) PlannerFactory(org.apache.druid.sql.calcite.planner.PlannerFactory) SqlLifecycleFactory(org.apache.druid.sql.SqlLifecycleFactory) Before(org.junit.Before)

Example 27 with Sequence

use of org.apache.druid.java.util.common.guava.Sequence in project druid by druid-io.

the class VectorizedVirtualColumnTest method testTimeseries.

private void testTimeseries(ColumnCapabilities capabilities) {
    testTimeseries(capabilities, CONTEXT_VECTORIZE_FORCE, true);
    TimeseriesQuery query = Druids.newTimeseriesQueryBuilder().intervals("2000/2030").dataSource(QueryRunnerTestHelper.DATA_SOURCE).granularity(Granularities.ALL).virtualColumns(new AlwaysTwoVectorizedVirtualColumn(ALWAYS_TWO, capabilities)).aggregators(new AlwaysTwoCounterAggregatorFactory(COUNT, ALWAYS_TWO)).context(CONTEXT_VECTORIZE_FORCE).build();
    Sequence seq = timeseriesTestHelper.runQueryOnSegmentsObjs(segments, query);
    List<Result<TimeseriesResultValue>> expectedResults = ImmutableList.of(new Result<>(DateTimes.of("2011-01-12T00:00:00.000Z"), new TimeseriesResultValue(ImmutableMap.of(COUNT, getCount(capabilities)))));
    TestHelper.assertExpectedObjects(expectedResults, seq.toList(), "failed");
}
Also used : TimeseriesResultValue(org.apache.druid.query.timeseries.TimeseriesResultValue) TimeseriesQuery(org.apache.druid.query.timeseries.TimeseriesQuery) Sequence(org.apache.druid.java.util.common.guava.Sequence) Result(org.apache.druid.query.Result)

Example 28 with Sequence

use of org.apache.druid.java.util.common.guava.Sequence in project druid by druid-io.

the class ExpressionVectorSelectorsTest method sanityTestVectorizedExpressionSelectors.

public static void sanityTestVectorizedExpressionSelectors(String expression, @Nullable ExpressionType outputType, QueryableIndex index, Closer closer, int rowsPerSegment) {
    final List<Object> results = new ArrayList<>(rowsPerSegment);
    final VirtualColumns virtualColumns = VirtualColumns.create(ImmutableList.of(new ExpressionVirtualColumn("v", expression, ExpressionType.toColumnType(outputType), TestExprMacroTable.INSTANCE)));
    final QueryableIndexStorageAdapter storageAdapter = new QueryableIndexStorageAdapter(index);
    VectorCursor cursor = storageAdapter.makeVectorCursor(null, index.getDataInterval(), virtualColumns, false, 512, null);
    ColumnCapabilities capabilities = virtualColumns.getColumnCapabilities(storageAdapter, "v");
    int rowCount = 0;
    if (capabilities.isDictionaryEncoded().isTrue()) {
        SingleValueDimensionVectorSelector selector = cursor.getColumnSelectorFactory().makeSingleValueDimensionSelector(DefaultDimensionSpec.of("v"));
        while (!cursor.isDone()) {
            int[] row = selector.getRowVector();
            for (int i = 0; i < selector.getCurrentVectorSize(); i++, rowCount++) {
                results.add(selector.lookupName(row[i]));
            }
            cursor.advance();
        }
    } else {
        VectorValueSelector selector = null;
        VectorObjectSelector objectSelector = null;
        if (outputType != null && outputType.isNumeric()) {
            selector = cursor.getColumnSelectorFactory().makeValueSelector("v");
        } else {
            objectSelector = cursor.getColumnSelectorFactory().makeObjectSelector("v");
        }
        while (!cursor.isDone()) {
            boolean[] nulls;
            switch(outputType.getType()) {
                case LONG:
                    nulls = selector.getNullVector();
                    long[] longs = selector.getLongVector();
                    for (int i = 0; i < selector.getCurrentVectorSize(); i++, rowCount++) {
                        results.add(nulls != null && nulls[i] ? null : longs[i]);
                    }
                    break;
                case DOUBLE:
                    // special case to test floats just to get coverage on getFloatVector
                    if ("float2".equals(expression)) {
                        nulls = selector.getNullVector();
                        float[] floats = selector.getFloatVector();
                        for (int i = 0; i < selector.getCurrentVectorSize(); i++, rowCount++) {
                            results.add(nulls != null && nulls[i] ? null : (double) floats[i]);
                        }
                    } else {
                        nulls = selector.getNullVector();
                        double[] doubles = selector.getDoubleVector();
                        for (int i = 0; i < selector.getCurrentVectorSize(); i++, rowCount++) {
                            results.add(nulls != null && nulls[i] ? null : doubles[i]);
                        }
                    }
                    break;
                case STRING:
                    Object[] objects = objectSelector.getObjectVector();
                    for (int i = 0; i < objectSelector.getCurrentVectorSize(); i++, rowCount++) {
                        results.add(objects[i]);
                    }
                    break;
            }
            cursor.advance();
        }
    }
    closer.register(cursor);
    Sequence<Cursor> cursors = new QueryableIndexStorageAdapter(index).makeCursors(null, index.getDataInterval(), virtualColumns, Granularities.ALL, false, null);
    int rowCountCursor = cursors.map(nonVectorized -> {
        final ColumnValueSelector nonSelector = nonVectorized.getColumnSelectorFactory().makeColumnValueSelector("v");
        int rows = 0;
        while (!nonVectorized.isDone()) {
            Assert.assertEquals(StringUtils.format("Failed at row %s", rows), nonSelector.getObject(), results.get(rows));
            rows++;
            nonVectorized.advance();
        }
        return rows;
    }).accumulate(0, (acc, in) -> acc + in);
    Assert.assertTrue(rowCountCursor > 0);
    Assert.assertEquals(rowCountCursor, rowCount);
}
Also used : BeforeClass(org.junit.BeforeClass) SegmentGenerator(org.apache.druid.segment.generator.SegmentGenerator) RunWith(org.junit.runner.RunWith) ColumnValueSelector(org.apache.druid.segment.ColumnValueSelector) Parser(org.apache.druid.math.expr.Parser) ArrayList(java.util.ArrayList) DefaultDimensionSpec(org.apache.druid.query.dimension.DefaultDimensionSpec) GeneratorBasicSchemas(org.apache.druid.segment.generator.GeneratorBasicSchemas) VectorCursor(org.apache.druid.segment.vector.VectorCursor) ImmutableList(com.google.common.collect.ImmutableList) ExpressionType(org.apache.druid.math.expr.ExpressionType) Expr(org.apache.druid.math.expr.Expr) SingleValueDimensionVectorSelector(org.apache.druid.segment.vector.SingleValueDimensionVectorSelector) Parameterized(org.junit.runners.Parameterized) Nullable(javax.annotation.Nullable) Before(org.junit.Before) Sequence(org.apache.druid.java.util.common.guava.Sequence) ColumnInspector(org.apache.druid.segment.ColumnInspector) AfterClass(org.junit.AfterClass) QueryableIndexStorageAdapter(org.apache.druid.segment.QueryableIndexStorageAdapter) VirtualColumns(org.apache.druid.segment.VirtualColumns) Closer(org.apache.druid.java.util.common.io.Closer) QueryableIndex(org.apache.druid.segment.QueryableIndex) StringUtils(org.apache.druid.java.util.common.StringUtils) GeneratorSchemaInfo(org.apache.druid.segment.generator.GeneratorSchemaInfo) VectorObjectSelector(org.apache.druid.segment.vector.VectorObjectSelector) Test(org.junit.Test) IOException(java.io.IOException) TestExprMacroTable(org.apache.druid.query.expression.TestExprMacroTable) VectorValueSelector(org.apache.druid.segment.vector.VectorValueSelector) Collectors(java.util.stream.Collectors) Granularities(org.apache.druid.java.util.common.granularity.Granularities) ExprMacroTable(org.apache.druid.math.expr.ExprMacroTable) List(java.util.List) Cursor(org.apache.druid.segment.Cursor) LinearShardSpec(org.apache.druid.timeline.partition.LinearShardSpec) DataSegment(org.apache.druid.timeline.DataSegment) ColumnCapabilities(org.apache.druid.segment.column.ColumnCapabilities) Assert(org.junit.Assert) ArrayList(java.util.ArrayList) QueryableIndexStorageAdapter(org.apache.druid.segment.QueryableIndexStorageAdapter) VectorCursor(org.apache.druid.segment.vector.VectorCursor) Cursor(org.apache.druid.segment.Cursor) VectorCursor(org.apache.druid.segment.vector.VectorCursor) ColumnCapabilities(org.apache.druid.segment.column.ColumnCapabilities) VectorValueSelector(org.apache.druid.segment.vector.VectorValueSelector) SingleValueDimensionVectorSelector(org.apache.druid.segment.vector.SingleValueDimensionVectorSelector) VectorObjectSelector(org.apache.druid.segment.vector.VectorObjectSelector) VirtualColumns(org.apache.druid.segment.VirtualColumns) ColumnValueSelector(org.apache.druid.segment.ColumnValueSelector)

Example 29 with Sequence

use of org.apache.druid.java.util.common.guava.Sequence in project druid by druid-io.

the class VectorizedVirtualColumnTest method testTimeseries.

private void testTimeseries(ColumnCapabilities capabilities, Map<String, Object> context, boolean canVectorize) {
    TimeseriesQuery query = Druids.newTimeseriesQueryBuilder().intervals("2000/2030").dataSource(QueryRunnerTestHelper.DATA_SOURCE).granularity(Granularities.ALL).virtualColumns(new AlwaysTwoVectorizedVirtualColumn(ALWAYS_TWO, capabilities, canVectorize)).aggregators(new AlwaysTwoCounterAggregatorFactory(COUNT, ALWAYS_TWO)).context(context).build();
    Sequence seq = timeseriesTestHelper.runQueryOnSegmentsObjs(segments, query);
    List<Result<TimeseriesResultValue>> expectedResults = ImmutableList.of(new Result<>(DateTimes.of("2011-01-12T00:00:00.000Z"), new TimeseriesResultValue(ImmutableMap.of(COUNT, getCount(capabilities)))));
    TestHelper.assertExpectedObjects(expectedResults, seq.toList(), "failed");
}
Also used : TimeseriesResultValue(org.apache.druid.query.timeseries.TimeseriesResultValue) TimeseriesQuery(org.apache.druid.query.timeseries.TimeseriesQuery) Sequence(org.apache.druid.java.util.common.guava.Sequence) Result(org.apache.druid.query.Result)

Example 30 with Sequence

use of org.apache.druid.java.util.common.guava.Sequence in project druid by druid-io.

the class TopNQueryEngine method query.

/**
 * Do the thing - process a {@link StorageAdapter} into a {@link Sequence} of {@link TopNResultValue}, with one of the
 * fine {@link TopNAlgorithm} available chosen based on the type of column being aggregated. The algorithm provides a
 * mapping function to process rows from the adapter {@link org.apache.druid.segment.Cursor} to apply
 * {@link AggregatorFactory} and create or update {@link TopNResultValue}
 */
public Sequence<Result<TopNResultValue>> query(final TopNQuery query, final StorageAdapter adapter, @Nullable final TopNQueryMetrics queryMetrics) {
    if (adapter == null) {
        throw new SegmentMissingException("Null storage adapter found. Probably trying to issue a query against a segment being memory unmapped.");
    }
    final List<Interval> queryIntervals = query.getQuerySegmentSpec().getIntervals();
    final Filter filter = Filters.convertToCNFFromQueryContext(query, Filters.toFilter(query.getDimensionsFilter()));
    final Granularity granularity = query.getGranularity();
    final TopNMapFn mapFn = getMapFn(query, adapter, queryMetrics);
    Preconditions.checkArgument(queryIntervals.size() == 1, "Can only handle a single interval, got[%s]", queryIntervals);
    return Sequences.filter(Sequences.map(adapter.makeCursors(filter, queryIntervals.get(0), query.getVirtualColumns(), granularity, query.isDescending(), queryMetrics), input -> {
        if (queryMetrics != null) {
            queryMetrics.cursor(input);
        }
        return mapFn.apply(input, queryMetrics);
    }), Predicates.notNull());
}
Also used : Sequence(org.apache.druid.java.util.common.guava.Sequence) Granularity(org.apache.druid.java.util.common.granularity.Granularity) NonBlockingPool(org.apache.druid.collections.NonBlockingPool) ExtractionFn(org.apache.druid.query.extraction.ExtractionFn) AggregatorFactory(org.apache.druid.query.aggregation.AggregatorFactory) ValueType(org.apache.druid.segment.column.ValueType) SegmentMissingException(org.apache.druid.segment.SegmentMissingException) StorageAdapter(org.apache.druid.segment.StorageAdapter) ByteBuffer(java.nio.ByteBuffer) Result(org.apache.druid.query.Result) Interval(org.joda.time.Interval) List(java.util.List) ColumnHolder(org.apache.druid.segment.column.ColumnHolder) Types(org.apache.druid.segment.column.Types) Predicates(com.google.common.base.Predicates) ColumnCapabilities(org.apache.druid.segment.column.ColumnCapabilities) Preconditions(com.google.common.base.Preconditions) Filters(org.apache.druid.segment.filter.Filters) Sequences(org.apache.druid.java.util.common.guava.Sequences) Nullable(javax.annotation.Nullable) Filter(org.apache.druid.query.filter.Filter) Filter(org.apache.druid.query.filter.Filter) SegmentMissingException(org.apache.druid.segment.SegmentMissingException) Granularity(org.apache.druid.java.util.common.granularity.Granularity) Interval(org.joda.time.Interval)

Aggregations

Sequence (org.apache.druid.java.util.common.guava.Sequence)102 Test (org.junit.Test)53 List (java.util.List)44 DefaultDimensionSpec (org.apache.druid.query.dimension.DefaultDimensionSpec)37 ResponseContext (org.apache.druid.query.context.ResponseContext)32 ImmutableList (com.google.common.collect.ImmutableList)29 Intervals (org.apache.druid.java.util.common.Intervals)28 Granularities (org.apache.druid.java.util.common.granularity.Granularities)28 QueryRunner (org.apache.druid.query.QueryRunner)28 ArrayList (java.util.ArrayList)27 VirtualColumns (org.apache.druid.segment.VirtualColumns)26 Cursor (org.apache.druid.segment.Cursor)25 QueryPlus (org.apache.druid.query.QueryPlus)24 Result (org.apache.druid.query.Result)24 NullHandling (org.apache.druid.common.config.NullHandling)22 InitializedNullHandlingTest (org.apache.druid.testing.InitializedNullHandlingTest)22 MultipleIntervalSegmentSpec (org.apache.druid.query.spec.MultipleIntervalSegmentSpec)21 QueryableIndexStorageAdapter (org.apache.druid.segment.QueryableIndexStorageAdapter)20 DataSegment (org.apache.druid.timeline.DataSegment)20 ImmutableMap (com.google.common.collect.ImmutableMap)18