Search in sources :

Example 6 with AllGranularity

use of org.apache.druid.java.util.common.granularity.AllGranularity in project druid by druid-io.

the class SeekableStreamIndexTaskRunnerAuthTest method setUp.

@Before
public void setUp() {
    // Create an AuthorizerMapper that only allows access to a Datasource resource
    AuthorizerMapper authorizerMapper = new AuthorizerMapper(null) {

        @Override
        public Authorizer getAuthorizer(String name) {
            return (authenticationResult, resource, action) -> {
                final String username = authenticationResult.getIdentity();
                // - or, Datasource Write User requests Write access
                if (resource.getType().equals(ResourceType.DATASOURCE)) {
                    return new Access((action == Action.READ && username.equals(Users.DATASOURCE_READ)) || (action == Action.WRITE && username.equals(Users.DATASOURCE_WRITE)));
                }
                // Do not allow access to any other resource
                return new Access(false);
            };
        }
    };
    DataSchema dataSchema = new DataSchema("datasource", new TimestampSpec(null, null, null), new DimensionsSpec(Collections.emptyList()), new AggregatorFactory[] {}, new ArbitraryGranularitySpec(new AllGranularity(), Collections.emptyList()), TransformSpec.NONE, null, null);
    SeekableStreamIndexTaskTuningConfig tuningConfig = mock(SeekableStreamIndexTaskTuningConfig.class);
    SeekableStreamIndexTaskIOConfig<String, String> ioConfig = new TestSeekableStreamIndexTaskIOConfig();
    // Initiliaze task and task runner
    SeekableStreamIndexTask<String, String, ByteEntity> indexTask = new TestSeekableStreamIndexTask("id", dataSchema, tuningConfig, ioConfig);
    taskRunner = new TestSeekableStreamIndexTaskRunner(indexTask, authorizerMapper);
}
Also used : TaskToolbox(org.apache.druid.indexing.common.TaskToolbox) StreamPartition(org.apache.druid.indexing.seekablestream.common.StreamPartition) ArbitraryGranularitySpec(org.apache.druid.segment.indexing.granularity.ArbitraryGranularitySpec) RecordSupplier(org.apache.druid.indexing.seekablestream.common.RecordSupplier) OrderedPartitionableRecord(org.apache.druid.indexing.seekablestream.common.OrderedPartitionableRecord) AuthorizerMapper(org.apache.druid.server.security.AuthorizerMapper) EasyMock.mock(org.easymock.EasyMock.mock) AllGranularity(org.apache.druid.java.util.common.granularity.AllGranularity) TimestampSpec(org.apache.druid.data.input.impl.TimestampSpec) AuthenticationResult(org.apache.druid.server.security.AuthenticationResult) HttpServletRequest(javax.servlet.http.HttpServletRequest) Map(java.util.Map) ForbiddenException(org.apache.druid.server.security.ForbiddenException) EasyMock.replay(org.easymock.EasyMock.replay) AuthConfig(org.apache.druid.server.security.AuthConfig) TypeReference(com.fasterxml.jackson.core.type.TypeReference) ExpectedException(org.junit.rules.ExpectedException) Nonnull(javax.annotation.Nonnull) Nullable(javax.annotation.Nullable) Before(org.junit.Before) DateTimes(org.apache.druid.java.util.common.DateTimes) Access(org.apache.druid.server.security.Access) ResourceType(org.apache.druid.server.security.ResourceType) AggregatorFactory(org.apache.druid.query.aggregation.AggregatorFactory) ObjectMapper(com.fasterxml.jackson.databind.ObjectMapper) Set(java.util.Set) CsvInputFormat(org.apache.druid.data.input.impl.CsvInputFormat) DimensionsSpec(org.apache.druid.data.input.impl.DimensionsSpec) Test(org.junit.Test) Action(org.apache.druid.server.security.Action) EasyMock(org.easymock.EasyMock) LockGranularity(org.apache.druid.indexing.common.LockGranularity) OrderedSequenceNumber(org.apache.druid.indexing.seekablestream.common.OrderedSequenceNumber) Consumer(java.util.function.Consumer) List(java.util.List) Rule(org.junit.Rule) TreeMap(java.util.TreeMap) ByteEntity(org.apache.druid.data.input.impl.ByteEntity) DataSchema(org.apache.druid.segment.indexing.DataSchema) Collections(java.util.Collections) TransformSpec(org.apache.druid.segment.transform.TransformSpec) Authorizer(org.apache.druid.server.security.Authorizer) ByteEntity(org.apache.druid.data.input.impl.ByteEntity) AllGranularity(org.apache.druid.java.util.common.granularity.AllGranularity) Access(org.apache.druid.server.security.Access) ArbitraryGranularitySpec(org.apache.druid.segment.indexing.granularity.ArbitraryGranularitySpec) DataSchema(org.apache.druid.segment.indexing.DataSchema) TimestampSpec(org.apache.druid.data.input.impl.TimestampSpec) AuthorizerMapper(org.apache.druid.server.security.AuthorizerMapper) DimensionsSpec(org.apache.druid.data.input.impl.DimensionsSpec) Before(org.junit.Before)

Example 7 with AllGranularity

use of org.apache.druid.java.util.common.granularity.AllGranularity in project druid by druid-io.

the class CalciteCorrelatedQueryTest method testCorrelatedSubqueryWithLeftFilter_leftDirectAccessDisabled.

@Test
@Parameters(source = QueryContextForJoinProvider.class)
public void testCorrelatedSubqueryWithLeftFilter_leftDirectAccessDisabled(Map<String, Object> queryContext) throws Exception {
    cannotVectorize();
    testQuery("select country, ANY_VALUE(\n" + "        select max(\"users\") from (\n" + "            select floor(__time to day), count(*) \"users\" from visits f where f.country = visits.country group by 1\n" + "        )\n" + "     ) as \"dailyVisits\"\n" + "from visits \n" + " where city = 'B' and __time between '2021-01-01 01:00:00' AND '2021-01-02 23:59:59'" + " group by 1", queryContext, ImmutableList.of(GroupByQuery.builder().setDataSource(join(new QueryDataSource(newScanQueryBuilder().dataSource(CalciteTests.USERVISITDATASOURCE).intervals(querySegmentSpec(Intervals.of("2021-01-01T01:00:00.000Z/2021-01-02T23:59:59.001Z"))).filters(selector("city", "B", null)).columns("__time", "city", "country").build()), new QueryDataSource(GroupByQuery.builder().setDataSource(GroupByQuery.builder().setDataSource(CalciteTests.USERVISITDATASOURCE).setQuerySegmentSpec(querySegmentSpec(Intervals.ETERNITY)).setVirtualColumns(new ExpressionVirtualColumn("v0", "timestamp_floor(\"__time\",'P1D',null,'UTC')", ColumnType.LONG, TestExprMacroTable.INSTANCE)).setDimFilter(not(selector("country", null, null))).setDimensions(new DefaultDimensionSpec("v0", "d0", ColumnType.LONG), new DefaultDimensionSpec("country", "d1")).setAggregatorSpecs(new CountAggregatorFactory("a0")).setContext(withTimestampResultContext(queryContext, "d0", Granularities.DAY)).setGranularity(new AllGranularity()).build()).setQuerySegmentSpec(querySegmentSpec(Intervals.ETERNITY)).setDimensions(new DefaultDimensionSpec("d1", "_d0")).setAggregatorSpecs(new LongMaxAggregatorFactory("_a0", "a0")).setGranularity(new AllGranularity()).setContext(queryContext).build()), "j0.", equalsCondition(makeColumnExpression("country"), makeColumnExpression("j0._d0")), JoinType.LEFT)).setQuerySegmentSpec(querySegmentSpec(Intervals.ETERNITY)).setDimensions(new DefaultDimensionSpec("country", "d0")).setAggregatorSpecs(new LongAnyAggregatorFactory("a0", "j0._a0")).setGranularity(new AllGranularity()).setContext(queryContext).build()), ImmutableList.of(new Object[] { "canada", 4L }));
}
Also used : ExpressionVirtualColumn(org.apache.druid.segment.virtual.ExpressionVirtualColumn) QueryDataSource(org.apache.druid.query.QueryDataSource) CountAggregatorFactory(org.apache.druid.query.aggregation.CountAggregatorFactory) LongAnyAggregatorFactory(org.apache.druid.query.aggregation.any.LongAnyAggregatorFactory) AllGranularity(org.apache.druid.java.util.common.granularity.AllGranularity) LongMaxAggregatorFactory(org.apache.druid.query.aggregation.LongMaxAggregatorFactory) DefaultDimensionSpec(org.apache.druid.query.dimension.DefaultDimensionSpec) Parameters(junitparams.Parameters) Test(org.junit.Test)

Example 8 with AllGranularity

use of org.apache.druid.java.util.common.granularity.AllGranularity in project druid by druid-io.

the class CalciteCorrelatedQueryTest method testCorrelatedSubqueryWithLeftFilter.

@Test
@Parameters(source = QueryContextForJoinProvider.class)
public void testCorrelatedSubqueryWithLeftFilter(Map<String, Object> queryContext) throws Exception {
    cannotVectorize();
    queryContext = withLeftDirectAccessEnabled(queryContext);
    testQuery("select country, ANY_VALUE(\n" + "        select max(\"users\") from (\n" + "            select floor(__time to day), count(*) \"users\" from visits f where f.country = visits.country group by 1\n" + "        )\n" + "     ) as \"dailyVisits\"\n" + "from visits \n" + " where city = 'B' and __time between '2021-01-01 01:00:00' AND '2021-01-02 23:59:59'" + " group by 1", queryContext, ImmutableList.of(GroupByQuery.builder().setDataSource(join(new TableDataSource(CalciteTests.USERVISITDATASOURCE), new QueryDataSource(GroupByQuery.builder().setDataSource(GroupByQuery.builder().setDataSource(CalciteTests.USERVISITDATASOURCE).setQuerySegmentSpec(querySegmentSpec(Intervals.ETERNITY)).setVirtualColumns(new ExpressionVirtualColumn("v0", "timestamp_floor(\"__time\",'P1D',null,'UTC')", ColumnType.LONG, TestExprMacroTable.INSTANCE)).setDimFilter(not(selector("country", null, null))).setDimensions(new DefaultDimensionSpec("v0", "d0", ColumnType.LONG), new DefaultDimensionSpec("country", "d1")).setAggregatorSpecs(new CountAggregatorFactory("a0")).setContext(withTimestampResultContext(queryContext, "d0", Granularities.DAY)).setGranularity(new AllGranularity()).build()).setQuerySegmentSpec(querySegmentSpec(Intervals.ETERNITY)).setDimensions(new DefaultDimensionSpec("d1", "_d0")).setAggregatorSpecs(new LongMaxAggregatorFactory("_a0", "a0")).setGranularity(new AllGranularity()).setContext(queryContext).build()), "j0.", equalsCondition(makeColumnExpression("country"), makeColumnExpression("j0._d0")), JoinType.LEFT, selector("city", "B", null))).setQuerySegmentSpec(querySegmentSpec(Intervals.of("2021-01-01T01:00:00.000Z/2021-01-02T23:59:59.001Z"))).setDimensions(new DefaultDimensionSpec("country", "d0")).setAggregatorSpecs(new LongAnyAggregatorFactory("a0", "j0._a0")).setGranularity(new AllGranularity()).setContext(queryContext).build()), ImmutableList.of(new Object[] { "canada", 4L }));
}
Also used : ExpressionVirtualColumn(org.apache.druid.segment.virtual.ExpressionVirtualColumn) TableDataSource(org.apache.druid.query.TableDataSource) QueryDataSource(org.apache.druid.query.QueryDataSource) CountAggregatorFactory(org.apache.druid.query.aggregation.CountAggregatorFactory) LongAnyAggregatorFactory(org.apache.druid.query.aggregation.any.LongAnyAggregatorFactory) AllGranularity(org.apache.druid.java.util.common.granularity.AllGranularity) LongMaxAggregatorFactory(org.apache.druid.query.aggregation.LongMaxAggregatorFactory) DefaultDimensionSpec(org.apache.druid.query.dimension.DefaultDimensionSpec) Parameters(junitparams.Parameters) Test(org.junit.Test)

Example 9 with AllGranularity

use of org.apache.druid.java.util.common.granularity.AllGranularity in project druid by druid-io.

the class CalciteCorrelatedQueryTest method testCorrelatedSubqueryWithCorrelatedQueryFilter_Scan.

@Test
@Parameters(source = QueryContextForJoinProvider.class)
public void testCorrelatedSubqueryWithCorrelatedQueryFilter_Scan(Map<String, Object> queryContext) throws Exception {
    cannotVectorize();
    queryContext = withLeftDirectAccessEnabled(queryContext);
    testQuery("select country, ANY_VALUE(\n" + "        select max(\"users\") from (\n" + "            select floor(__time to day), count(user) \"users\" from visits f where f.country = visits.country and f.city = 'A' group by 1\n" + "        )\n" + "     ) as \"dailyVisits\"\n" + "from visits \n" + " where city = 'B'" + " group by 1", queryContext, ImmutableList.of(GroupByQuery.builder().setDataSource(join(new TableDataSource(CalciteTests.USERVISITDATASOURCE), new QueryDataSource(GroupByQuery.builder().setDataSource(GroupByQuery.builder().setDataSource(CalciteTests.USERVISITDATASOURCE).setQuerySegmentSpec(querySegmentSpec(Intervals.ETERNITY)).setVirtualColumns(new ExpressionVirtualColumn("v0", "timestamp_floor(\"__time\",'P1D',null,'UTC')", ColumnType.LONG, TestExprMacroTable.INSTANCE)).setDimensions(new DefaultDimensionSpec("v0", "d0", ColumnType.LONG), new DefaultDimensionSpec("country", "d1")).setAggregatorSpecs(new FilteredAggregatorFactory(new CountAggregatorFactory("a0"), not(selector("user", null, null)))).setDimFilter(and(selector("city", "A", null), not(selector("country", null, null)))).setContext(withTimestampResultContext(queryContext, "d0", Granularities.DAY)).setGranularity(new AllGranularity()).build()).setQuerySegmentSpec(querySegmentSpec(Intervals.ETERNITY)).setDimensions(new DefaultDimensionSpec("d1", "_d0")).setAggregatorSpecs(new LongMaxAggregatorFactory("_a0", "a0")).setGranularity(new AllGranularity()).setContext(queryContext).build()), "j0.", equalsCondition(makeColumnExpression("country"), makeColumnExpression("j0._d0")), JoinType.LEFT, selector("city", "B", null))).setQuerySegmentSpec(querySegmentSpec(Intervals.ETERNITY)).setDimensions(new DefaultDimensionSpec("country", "d0")).setAggregatorSpecs(new LongAnyAggregatorFactory("a0", "j0._a0")).setGranularity(new AllGranularity()).setContext(queryContext).build()), ImmutableList.of(new Object[] { "canada", 2L }));
}
Also used : FilteredAggregatorFactory(org.apache.druid.query.aggregation.FilteredAggregatorFactory) ExpressionVirtualColumn(org.apache.druid.segment.virtual.ExpressionVirtualColumn) TableDataSource(org.apache.druid.query.TableDataSource) QueryDataSource(org.apache.druid.query.QueryDataSource) CountAggregatorFactory(org.apache.druid.query.aggregation.CountAggregatorFactory) LongAnyAggregatorFactory(org.apache.druid.query.aggregation.any.LongAnyAggregatorFactory) AllGranularity(org.apache.druid.java.util.common.granularity.AllGranularity) LongMaxAggregatorFactory(org.apache.druid.query.aggregation.LongMaxAggregatorFactory) DefaultDimensionSpec(org.apache.druid.query.dimension.DefaultDimensionSpec) Parameters(junitparams.Parameters) Test(org.junit.Test)

Aggregations

AllGranularity (org.apache.druid.java.util.common.granularity.AllGranularity)9 Test (org.junit.Test)6 Parameters (junitparams.Parameters)5 QueryDataSource (org.apache.druid.query.QueryDataSource)5 CountAggregatorFactory (org.apache.druid.query.aggregation.CountAggregatorFactory)5 LongAnyAggregatorFactory (org.apache.druid.query.aggregation.any.LongAnyAggregatorFactory)5 DefaultDimensionSpec (org.apache.druid.query.dimension.DefaultDimensionSpec)5 ExpressionVirtualColumn (org.apache.druid.segment.virtual.ExpressionVirtualColumn)5 TableDataSource (org.apache.druid.query.TableDataSource)4 LongMaxAggregatorFactory (org.apache.druid.query.aggregation.LongMaxAggregatorFactory)4 AggregatorFactory (org.apache.druid.query.aggregation.AggregatorFactory)3 FilteredAggregatorFactory (org.apache.druid.query.aggregation.FilteredAggregatorFactory)3 LinkedHashMap (java.util.LinkedHashMap)2 TypeReference (com.fasterxml.jackson.core.type.TypeReference)1 ObjectMapper (com.fasterxml.jackson.databind.ObjectMapper)1 Collections (java.util.Collections)1 List (java.util.List)1 Map (java.util.Map)1 Set (java.util.Set)1 TreeMap (java.util.TreeMap)1