Search in sources :

Example 1 with LazySequence

use of org.apache.druid.java.util.common.guava.LazySequence in project druid by druid-io.

the class SqlResourceTest method setUp.

@Before
public void setUp() throws Exception {
    final QueryScheduler scheduler = new QueryScheduler(5, ManualQueryPrioritizationStrategy.INSTANCE, new HiLoQueryLaningStrategy(40), new ServerConfig()) {

        @Override
        public <T> Sequence<T> run(Query<?> query, Sequence<T> resultSequence) {
            return super.run(query, new LazySequence<T>(() -> {
                if (sleep) {
                    try {
                        // pretend to be a query that is waiting on results
                        Thread.sleep(500);
                    } catch (InterruptedException ignored) {
                    }
                }
                return resultSequence;
            }));
        }
    };
    executorService = MoreExecutors.listeningDecorator(Execs.multiThreaded(8, "test_sql_resource_%s"));
    walker = CalciteTests.createMockWalker(conglomerate, temporaryFolder.newFolder(), scheduler);
    final PlannerConfig plannerConfig = new PlannerConfig() {

        @Override
        public boolean shouldSerializeComplexValues() {
            return false;
        }
    };
    final DruidSchemaCatalog rootSchema = CalciteTests.createMockRootSchema(conglomerate, walker, plannerConfig, CalciteTests.TEST_AUTHORIZER_MAPPER);
    final DruidOperatorTable operatorTable = CalciteTests.createOperatorTable();
    final ExprMacroTable macroTable = CalciteTests.createExprMacroTable();
    req = EasyMock.createStrictMock(HttpServletRequest.class);
    EasyMock.expect(req.getRemoteAddr()).andReturn(null).once();
    EasyMock.expect(req.getAttribute(AuthConfig.DRUID_AUTHENTICATION_RESULT)).andReturn(CalciteTests.REGULAR_USER_AUTH_RESULT).anyTimes();
    EasyMock.expect(req.getAttribute(AuthConfig.DRUID_ALLOW_UNSECURED_PATH)).andReturn(null).anyTimes();
    EasyMock.expect(req.getAttribute(AuthConfig.DRUID_AUTHORIZATION_CHECKED)).andReturn(null).anyTimes();
    EasyMock.expect(req.getAttribute(AuthConfig.DRUID_AUTHENTICATION_RESULT)).andReturn(CalciteTests.REGULAR_USER_AUTH_RESULT).anyTimes();
    req.setAttribute(AuthConfig.DRUID_AUTHORIZATION_CHECKED, true);
    EasyMock.expectLastCall().anyTimes();
    EasyMock.expect(req.getAttribute(AuthConfig.DRUID_AUTHENTICATION_RESULT)).andReturn(CalciteTests.REGULAR_USER_AUTH_RESULT).anyTimes();
    EasyMock.replay(req);
    testRequestLogger = new TestRequestLogger();
    final PlannerFactory plannerFactory = new PlannerFactory(rootSchema, CalciteTests.createMockQueryMakerFactory(walker, conglomerate), operatorTable, macroTable, plannerConfig, CalciteTests.TEST_AUTHORIZER_MAPPER, CalciteTests.getJsonMapper(), CalciteTests.DRUID_SCHEMA_NAME);
    lifecycleManager = new SqlLifecycleManager() {

        @Override
        public void add(String sqlQueryId, SqlLifecycle lifecycle) {
            super.add(sqlQueryId, lifecycle);
            if (lifecycleAddLatch != null) {
                lifecycleAddLatch.countDown();
            }
        }
    };
    final ServiceEmitter emitter = new NoopServiceEmitter();
    sqlLifecycleFactory = new SqlLifecycleFactory(plannerFactory, emitter, testRequestLogger, scheduler) {

        @Override
        public SqlLifecycle factorize() {
            return new TestSqlLifecycle(plannerFactory, emitter, testRequestLogger, scheduler, System.currentTimeMillis(), System.nanoTime(), validateAndAuthorizeLatchSupplier, planLatchSupplier, executeLatchSupplier, sequenceMapFnSupplier);
        }
    };
    resource = new SqlResource(JSON_MAPPER, CalciteTests.TEST_AUTHORIZER_MAPPER, sqlLifecycleFactory, lifecycleManager, new ServerConfig());
}
Also used : SqlLifecycleManager(org.apache.druid.sql.SqlLifecycleManager) ServiceEmitter(org.apache.druid.java.util.emitter.service.ServiceEmitter) NoopServiceEmitter(org.apache.druid.server.metrics.NoopServiceEmitter) QueryScheduler(org.apache.druid.server.QueryScheduler) BaseQuery(org.apache.druid.query.BaseQuery) Query(org.apache.druid.query.Query) HiLoQueryLaningStrategy(org.apache.druid.server.scheduling.HiLoQueryLaningStrategy) SqlLifecycle(org.apache.druid.sql.SqlLifecycle) NoopServiceEmitter(org.apache.druid.server.metrics.NoopServiceEmitter) Sequence(org.apache.druid.java.util.common.guava.Sequence) LazySequence(org.apache.druid.java.util.common.guava.LazySequence) QueryInterruptedException(org.apache.druid.query.QueryInterruptedException) DruidOperatorTable(org.apache.druid.sql.calcite.planner.DruidOperatorTable) ExprMacroTable(org.apache.druid.math.expr.ExprMacroTable) TestRequestLogger(org.apache.druid.server.log.TestRequestLogger) HttpServletRequest(javax.servlet.http.HttpServletRequest) ServerConfig(org.apache.druid.server.initialization.ServerConfig) PlannerConfig(org.apache.druid.sql.calcite.planner.PlannerConfig) DruidSchemaCatalog(org.apache.druid.sql.calcite.schema.DruidSchemaCatalog) PlannerFactory(org.apache.druid.sql.calcite.planner.PlannerFactory) SqlLifecycleFactory(org.apache.druid.sql.SqlLifecycleFactory) Before(org.junit.Before)

Example 2 with LazySequence

use of org.apache.druid.java.util.common.guava.LazySequence in project druid by druid-io.

the class QueryResourceTest method createScheduledQueryResource.

private void createScheduledQueryResource(QueryScheduler scheduler, Collection<CountDownLatch> beforeScheduler, Collection<CountDownLatch> inScheduler) {
    QuerySegmentWalker texasRanger = new QuerySegmentWalker() {

        @Override
        public <T> QueryRunner<T> getQueryRunnerForIntervals(Query<T> query, Iterable<Interval> intervals) {
            return (queryPlus, responseContext) -> {
                beforeScheduler.forEach(CountDownLatch::countDown);
                return scheduler.run(scheduler.prioritizeAndLaneQuery(queryPlus, ImmutableSet.of()), new LazySequence<T>(() -> {
                    inScheduler.forEach(CountDownLatch::countDown);
                    try {
                        // pretend to be a query that is waiting on results
                        Thread.sleep(500);
                    } catch (InterruptedException ignored) {
                    }
                    // all that waiting for nothing :(
                    return Sequences.empty();
                }));
            };
        }

        @Override
        public <T> QueryRunner<T> getQueryRunnerForSegments(Query<T> query, Iterable<SegmentDescriptor> specs) {
            return getQueryRunnerForIntervals(null, null);
        }
    };
    queryResource = new QueryResource(new QueryLifecycleFactory(WAREHOUSE, texasRanger, new DefaultGenericQueryMetricsFactory(), new NoopServiceEmitter(), testRequestLogger, new AuthConfig(), AuthTestUtils.TEST_AUTHORIZER_MAPPER, Suppliers.ofInstance(new DefaultQueryConfig(ImmutableMap.of()))), jsonMapper, smileMapper, scheduler, new AuthConfig(), null, ResponseContextConfig.newConfig(true), DRUID_NODE);
}
Also used : TimeBoundaryResultValue(org.apache.druid.query.timeboundary.TimeBoundaryResultValue) AuthorizerMapper(org.apache.druid.server.security.AuthorizerMapper) Key(com.google.inject.Key) Smile(org.apache.druid.guice.annotations.Smile) DefaultQueryConfig(org.apache.druid.query.DefaultQueryConfig) HttpStatus(org.apache.http.HttpStatus) MediaType(javax.ws.rs.core.MediaType) ByteArrayInputStream(java.io.ByteArrayInputStream) BadJsonQueryException(org.apache.druid.query.BadJsonQueryException) After(org.junit.After) QueryRunner(org.apache.druid.query.QueryRunner) QueryToolChestWarehouse(org.apache.druid.query.QueryToolChestWarehouse) ForbiddenException(org.apache.druid.server.security.ForbiddenException) AuthConfig(org.apache.druid.server.security.AuthConfig) TypeReference(com.fasterxml.jackson.core.type.TypeReference) ManualQueryPrioritizationStrategy(org.apache.druid.server.scheduling.ManualQueryPrioritizationStrategy) HiLoQueryLaningStrategy(org.apache.druid.server.scheduling.HiLoQueryLaningStrategy) TestRequestLogger(org.apache.druid.server.log.TestRequestLogger) SmileMediaTypes(com.fasterxml.jackson.jaxrs.smile.SmileMediaTypes) LazySequence(org.apache.druid.java.util.common.guava.LazySequence) ImmutableSet(com.google.common.collect.ImmutableSet) Execs(org.apache.druid.java.util.common.concurrent.Execs) ImmutableMap(com.google.common.collect.ImmutableMap) Collection(java.util.Collection) MapQueryToolChestWarehouse(org.apache.druid.query.MapQueryToolChestWarehouse) StreamingOutput(javax.ws.rs.core.StreamingOutput) Action(org.apache.druid.server.security.Action) GuiceInjectors(org.apache.druid.guice.GuiceInjectors) StandardCharsets(java.nio.charset.StandardCharsets) Executors(java.util.concurrent.Executors) CountDownLatch(java.util.concurrent.CountDownLatch) List(java.util.List) QueryTimeoutException(org.apache.druid.query.QueryTimeoutException) Response(javax.ws.rs.core.Response) ServiceEmitter(org.apache.druid.java.util.emitter.service.ServiceEmitter) Authorizer(org.apache.druid.server.security.Authorizer) MoreExecutors(com.google.common.util.concurrent.MoreExecutors) ListenableFuture(com.google.common.util.concurrent.ListenableFuture) QueryException(org.apache.druid.query.QueryException) BeforeClass(org.junit.BeforeClass) ByteArrayOutputStream(java.io.ByteArrayOutputStream) QueryCapacityExceededException(org.apache.druid.query.QueryCapacityExceededException) AuthenticationResult(org.apache.druid.server.security.AuthenticationResult) Interval(org.joda.time.Interval) HttpServletRequest(javax.servlet.http.HttpServletRequest) ImmutableList(com.google.common.collect.ImmutableList) Query(org.apache.druid.query.Query) Suppliers(com.google.common.base.Suppliers) AuthTestUtils(org.apache.druid.server.security.AuthTestUtils) QuerySegmentWalker(org.apache.druid.query.QuerySegmentWalker) Status(javax.ws.rs.core.Response.Status) Sequences(org.apache.druid.java.util.common.guava.Sequences) NoopServiceEmitter(org.apache.druid.server.metrics.NoopServiceEmitter) Before(org.junit.Before) Access(org.apache.druid.server.security.Access) EmittingLogger(org.apache.druid.java.util.emitter.EmittingLogger) ServerConfig(org.apache.druid.server.initialization.ServerConfig) QueryInterruptedException(org.apache.druid.query.QueryInterruptedException) ObjectMapper(com.fasterxml.jackson.databind.ObjectMapper) Test(org.junit.Test) IOException(java.io.IOException) EasyMock(org.easymock.EasyMock) DefaultObjectMapper(org.apache.druid.jackson.DefaultObjectMapper) Injector(com.google.inject.Injector) Consumer(java.util.function.Consumer) Result(org.apache.druid.query.Result) ThresholdBasedQueryPrioritizationStrategy(org.apache.druid.server.scheduling.ThresholdBasedQueryPrioritizationStrategy) TruncatedResponseContextException(org.apache.druid.query.TruncatedResponseContextException) Resource(org.apache.druid.server.security.Resource) ResourceLimitExceededException(org.apache.druid.query.ResourceLimitExceededException) NoQueryLaningStrategy(org.apache.druid.server.scheduling.NoQueryLaningStrategy) DefaultGenericQueryMetricsFactory(org.apache.druid.query.DefaultGenericQueryMetricsFactory) SegmentDescriptor(org.apache.druid.query.SegmentDescriptor) Assert(org.junit.Assert) Collections(java.util.Collections) QueryUnsupportedException(org.apache.druid.query.QueryUnsupportedException) Query(org.apache.druid.query.Query) NoopServiceEmitter(org.apache.druid.server.metrics.NoopServiceEmitter) AuthConfig(org.apache.druid.server.security.AuthConfig) CountDownLatch(java.util.concurrent.CountDownLatch) QueryInterruptedException(org.apache.druid.query.QueryInterruptedException) QuerySegmentWalker(org.apache.druid.query.QuerySegmentWalker) DefaultQueryConfig(org.apache.druid.query.DefaultQueryConfig) DefaultGenericQueryMetricsFactory(org.apache.druid.query.DefaultGenericQueryMetricsFactory) LazySequence(org.apache.druid.java.util.common.guava.LazySequence)

Example 3 with LazySequence

use of org.apache.druid.java.util.common.guava.LazySequence in project druid by druid-io.

the class TestClusterQuerySegmentWalker method getQueryRunnerForSegments.

@Override
public <T> QueryRunner<T> getQueryRunnerForSegments(final Query<T> query, final Iterable<SegmentDescriptor> specs) {
    final QueryRunnerFactory<T, Query<T>> factory = conglomerate.findFactory(query);
    if (factory == null) {
        throw new ISE("Unknown query type[%s].", query.getClass());
    }
    final DataSourceAnalysis analysis = DataSourceAnalysis.forDataSource(query.getDataSource());
    if (!analysis.isConcreteTableBased()) {
        throw new ISE("Cannot handle datasource: %s", query.getDataSource());
    }
    final String dataSourceName = ((TableDataSource) analysis.getBaseDataSource()).getName();
    final QueryToolChest<T, Query<T>> toolChest = factory.getToolchest();
    // Make sure this query type can handle the subquery, if present.
    if (analysis.isQuery() && !toolChest.canPerformSubquery(((QueryDataSource) analysis.getDataSource()).getQuery())) {
        throw new ISE("Cannot handle subquery: %s", analysis.getDataSource());
    }
    final Function<SegmentReference, SegmentReference> segmentMapFn = joinableFactoryWrapper.createSegmentMapFn(analysis.getJoinBaseTableFilter().map(Filters::toFilter).orElse(null), analysis.getPreJoinableClauses(), new AtomicLong(), analysis.getBaseQuery().orElse(query));
    final QueryRunner<T> baseRunner = new FinalizeResultsQueryRunner<>(toolChest.postMergeQueryDecoration(toolChest.mergeResults(toolChest.preMergeQueryDecoration(makeTableRunner(toolChest, factory, getSegmentsForTable(dataSourceName, specs), segmentMapFn)))), toolChest);
    // to actually serve the queries
    return (theQuery, responseContext) -> {
        responseContext.initializeRemainingResponses();
        responseContext.addRemainingResponse(theQuery.getQuery().getMostSpecificId(), 0);
        if (scheduler != null) {
            Set<SegmentServerSelector> segments = new HashSet<>();
            specs.forEach(spec -> segments.add(new SegmentServerSelector(spec)));
            return scheduler.run(scheduler.prioritizeAndLaneQuery(theQuery, segments), new LazySequence<>(() -> baseRunner.run(theQuery.withQuery(Queries.withSpecificSegments(theQuery.getQuery(), ImmutableList.copyOf(specs))), responseContext)));
        } else {
            return baseRunner.run(theQuery.withQuery(Queries.withSpecificSegments(theQuery.getQuery(), ImmutableList.copyOf(specs))), responseContext);
        }
    };
}
Also used : DataSourceAnalysis(org.apache.druid.query.planning.DataSourceAnalysis) JoinableFactory(org.apache.druid.segment.join.JoinableFactory) Function(java.util.function.Function) NoopQueryRunner(org.apache.druid.query.NoopQueryRunner) ArrayList(java.util.ArrayList) HashSet(java.util.HashSet) SegmentReference(org.apache.druid.segment.SegmentReference) PartitionChunk(org.apache.druid.timeline.partition.PartitionChunk) Interval(org.joda.time.Interval) Lists(com.google.common.collect.Lists) ImmutableList(com.google.common.collect.ImmutableList) Query(org.apache.druid.query.Query) SpecificSegmentQueryRunner(org.apache.druid.query.spec.SpecificSegmentQueryRunner) Map(java.util.Map) SegmentServerSelector(org.apache.druid.client.SegmentServerSelector) QueryRunner(org.apache.druid.query.QueryRunner) QuerySegmentWalker(org.apache.druid.query.QuerySegmentWalker) Nullable(javax.annotation.Nullable) FinalizeResultsQueryRunner(org.apache.druid.query.FinalizeResultsQueryRunner) LazySequence(org.apache.druid.java.util.common.guava.LazySequence) Execs(org.apache.druid.java.util.common.concurrent.Execs) VersionedIntervalTimeline(org.apache.druid.timeline.VersionedIntervalTimeline) QueryRunnerFactoryConglomerate(org.apache.druid.query.QueryRunnerFactoryConglomerate) QueryToolChest(org.apache.druid.query.QueryToolChest) JoinableFactoryWrapper(org.apache.druid.segment.join.JoinableFactoryWrapper) TimelineObjectHolder(org.apache.druid.timeline.TimelineObjectHolder) Set(java.util.Set) ISE(org.apache.druid.java.util.common.ISE) SpecificSegmentSpec(org.apache.druid.query.spec.SpecificSegmentSpec) ReferenceCountingSegment(org.apache.druid.segment.ReferenceCountingSegment) TableDataSource(org.apache.druid.query.TableDataSource) Queries(org.apache.druid.query.Queries) AtomicLong(java.util.concurrent.atomic.AtomicLong) QueryDataSource(org.apache.druid.query.QueryDataSource) List(java.util.List) ReferenceCountingSegmentQueryRunner(org.apache.druid.query.ReferenceCountingSegmentQueryRunner) QueryRunnerFactory(org.apache.druid.query.QueryRunnerFactory) Preconditions(com.google.common.base.Preconditions) FunctionalIterable(org.apache.druid.java.util.common.guava.FunctionalIterable) SegmentDescriptor(org.apache.druid.query.SegmentDescriptor) Filters(org.apache.druid.segment.filter.Filters) Collections(java.util.Collections) HashSet(java.util.HashSet) Set(java.util.Set) Query(org.apache.druid.query.Query) SegmentReference(org.apache.druid.segment.SegmentReference) DataSourceAnalysis(org.apache.druid.query.planning.DataSourceAnalysis) Filters(org.apache.druid.segment.filter.Filters) AtomicLong(java.util.concurrent.atomic.AtomicLong) TableDataSource(org.apache.druid.query.TableDataSource) FinalizeResultsQueryRunner(org.apache.druid.query.FinalizeResultsQueryRunner) SegmentServerSelector(org.apache.druid.client.SegmentServerSelector) ISE(org.apache.druid.java.util.common.ISE) LazySequence(org.apache.druid.java.util.common.guava.LazySequence)

Aggregations

LazySequence (org.apache.druid.java.util.common.guava.LazySequence)3 Query (org.apache.druid.query.Query)3 ImmutableList (com.google.common.collect.ImmutableList)2 Collections (java.util.Collections)2 List (java.util.List)2 HttpServletRequest (javax.servlet.http.HttpServletRequest)2 Execs (org.apache.druid.java.util.common.concurrent.Execs)2 QueryRunner (org.apache.druid.query.QueryRunner)2 QuerySegmentWalker (org.apache.druid.query.QuerySegmentWalker)2 SegmentDescriptor (org.apache.druid.query.SegmentDescriptor)2 Interval (org.joda.time.Interval)2 TypeReference (com.fasterxml.jackson.core.type.TypeReference)1 ObjectMapper (com.fasterxml.jackson.databind.ObjectMapper)1 SmileMediaTypes (com.fasterxml.jackson.jaxrs.smile.SmileMediaTypes)1 Preconditions (com.google.common.base.Preconditions)1 Suppliers (com.google.common.base.Suppliers)1 ImmutableMap (com.google.common.collect.ImmutableMap)1 ImmutableSet (com.google.common.collect.ImmutableSet)1 Lists (com.google.common.collect.Lists)1 ListenableFuture (com.google.common.util.concurrent.ListenableFuture)1