Search in sources :

Example 56 with DefaultObjectMapper

use of org.apache.druid.jackson.DefaultObjectMapper in project druid by druid-io.

the class BroadcastJoinableMMappedQueryableSegmentizerFactoryTest method testSegmentizer.

@Test
public void testSegmentizer() throws IOException, SegmentLoadingException {
    final ObjectMapper mapper = new DefaultObjectMapper();
    mapper.registerModule(new SegmentizerModule());
    final IndexIO indexIO = new IndexIO(mapper, () -> 0);
    mapper.setInjectableValues(new InjectableValues.Std().addValue(ExprMacroTable.class.getName(), TestExprMacroTable.INSTANCE).addValue(ObjectMapper.class.getName(), mapper).addValue(IndexIO.class, indexIO).addValue(DataSegment.PruneSpecsHolder.class, DataSegment.PruneSpecsHolder.DEFAULT));
    IndexMerger indexMerger = new IndexMergerV9(mapper, indexIO, OffHeapMemorySegmentWriteOutMediumFactory.instance());
    SegmentizerFactory expectedFactory = new BroadcastJoinableMMappedQueryableSegmentizerFactory(indexIO, KEY_COLUMNS);
    Interval testInterval = Intervals.of("2011-01-12T00:00:00.000Z/2011-05-01T00:00:00.000Z");
    IncrementalIndex data = TestIndex.makeRealtimeIndex("druid.sample.numeric.tsv");
    List<String> columnNames = data.getColumnNames();
    File segment = new File(temporaryFolder.newFolder(), "segment");
    File persistedSegmentRoot = indexMerger.persist(data, testInterval, segment, new IndexSpec(null, null, null, null, expectedFactory), null);
    File factoryJson = new File(persistedSegmentRoot, "factory.json");
    Assert.assertTrue(factoryJson.exists());
    SegmentizerFactory factory = mapper.readValue(factoryJson, SegmentizerFactory.class);
    Assert.assertTrue(factory instanceof BroadcastJoinableMMappedQueryableSegmentizerFactory);
    Assert.assertEquals(expectedFactory, factory);
    // load a segment
    final DataSegment dataSegment = new DataSegment(TABLE_NAME, testInterval, DateTimes.nowUtc().toString(), ImmutableMap.of(), columnNames, ImmutableList.of(), null, null, persistedSegmentRoot.getTotalSpace());
    final Segment loaded = factory.factorize(dataSegment, persistedSegmentRoot, false, SegmentLazyLoadFailCallback.NOOP);
    final BroadcastSegmentIndexedTable table = (BroadcastSegmentIndexedTable) loaded.as(IndexedTable.class);
    Assert.assertNotNull(table);
}
Also used : IndexMerger(org.apache.druid.segment.IndexMerger) IndexSpec(org.apache.druid.segment.IndexSpec) IncrementalIndex(org.apache.druid.segment.incremental.IncrementalIndex) IndexMergerV9(org.apache.druid.segment.IndexMergerV9) BroadcastSegmentIndexedTable(org.apache.druid.segment.join.table.BroadcastSegmentIndexedTable) IndexedTable(org.apache.druid.segment.join.table.IndexedTable) InjectableValues(com.fasterxml.jackson.databind.InjectableValues) DataSegment(org.apache.druid.timeline.DataSegment) TestExprMacroTable(org.apache.druid.query.expression.TestExprMacroTable) ExprMacroTable(org.apache.druid.math.expr.ExprMacroTable) Segment(org.apache.druid.segment.Segment) DataSegment(org.apache.druid.timeline.DataSegment) IndexIO(org.apache.druid.segment.IndexIO) SegmentizerModule(org.apache.druid.jackson.SegmentizerModule) DefaultObjectMapper(org.apache.druid.jackson.DefaultObjectMapper) BroadcastSegmentIndexedTable(org.apache.druid.segment.join.table.BroadcastSegmentIndexedTable) File(java.io.File) ObjectMapper(com.fasterxml.jackson.databind.ObjectMapper) DefaultObjectMapper(org.apache.druid.jackson.DefaultObjectMapper) Interval(org.joda.time.Interval) InitializedNullHandlingTest(org.apache.druid.testing.InitializedNullHandlingTest) Test(org.junit.Test)

Example 57 with DefaultObjectMapper

use of org.apache.druid.jackson.DefaultObjectMapper in project druid by druid-io.

the class QueryResourceTest method testSecuredCancelQuery.

@Test(timeout = 60_000L)
public void testSecuredCancelQuery() throws Exception {
    final CountDownLatch waitForCancellationLatch = new CountDownLatch(1);
    final CountDownLatch waitFinishLatch = new CountDownLatch(2);
    final CountDownLatch startAwaitLatch = new CountDownLatch(1);
    final CountDownLatch cancelledCountDownLatch = new CountDownLatch(1);
    EasyMock.expect(testServletRequest.getAttribute(AuthConfig.DRUID_AUTHORIZATION_CHECKED)).andReturn(null).anyTimes();
    EasyMock.expect(testServletRequest.getAttribute(AuthConfig.DRUID_ALLOW_UNSECURED_PATH)).andReturn(null).anyTimes();
    EasyMock.expect(testServletRequest.getAttribute(AuthConfig.DRUID_AUTHENTICATION_RESULT)).andReturn(AUTHENTICATION_RESULT).anyTimes();
    testServletRequest.setAttribute(AuthConfig.DRUID_AUTHORIZATION_CHECKED, true);
    EasyMock.expectLastCall().times(1);
    EasyMock.replay(testServletRequest);
    AuthorizerMapper authMapper = new AuthorizerMapper(null) {

        @Override
        public Authorizer getAuthorizer(String name) {
            return new Authorizer() {

                @Override
                public Access authorize(AuthenticationResult authenticationResult, Resource resource, Action action) {
                    // WRITE corresponds to cancellation of query
                    if (action.equals(Action.READ)) {
                        try {
                            // Countdown startAwaitLatch as we want query cancellation to happen
                            // after we enter isAuthorized method so that we can handle the
                            // InterruptedException here because of query cancellation
                            startAwaitLatch.countDown();
                            waitForCancellationLatch.await();
                        } catch (InterruptedException e) {
                            // When the query is cancelled the control will reach here,
                            // countdown the latch and rethrow the exception so that error response is returned for the query
                            cancelledCountDownLatch.countDown();
                            throw new RuntimeException(e);
                        }
                        return new Access(true);
                    } else {
                        return new Access(true);
                    }
                }
            };
        }
    };
    queryResource = new QueryResource(new QueryLifecycleFactory(WAREHOUSE, TEST_SEGMENT_WALKER, new DefaultGenericQueryMetricsFactory(), new NoopServiceEmitter(), testRequestLogger, new AuthConfig(), authMapper, Suppliers.ofInstance(new DefaultQueryConfig(ImmutableMap.of()))), jsonMapper, smileMapper, queryScheduler, new AuthConfig(), authMapper, ResponseContextConfig.newConfig(true), DRUID_NODE);
    final String queryString = "{\"queryType\":\"timeBoundary\", \"dataSource\":\"allow\"," + "\"context\":{\"queryId\":\"id_1\"}}";
    ObjectMapper mapper = new DefaultObjectMapper();
    Query<?> query = mapper.readValue(queryString, Query.class);
    ListenableFuture<?> future = MoreExecutors.listeningDecorator(Execs.singleThreaded("test_query_resource_%s")).submit(new Runnable() {

        @Override
        public void run() {
            try {
                Response response = queryResource.doPost(new ByteArrayInputStream(queryString.getBytes(StandardCharsets.UTF_8)), null, testServletRequest);
                Assert.assertEquals(Response.Status.INTERNAL_SERVER_ERROR.getStatusCode(), response.getStatus());
            } catch (IOException e) {
                throw new RuntimeException(e);
            }
            waitFinishLatch.countDown();
        }
    });
    queryScheduler.registerQueryFuture(query, future);
    startAwaitLatch.await();
    Executors.newSingleThreadExecutor().submit(new Runnable() {

        @Override
        public void run() {
            Response response = queryResource.cancelQuery("id_1", testServletRequest);
            Assert.assertEquals(Response.Status.ACCEPTED.getStatusCode(), response.getStatus());
            waitForCancellationLatch.countDown();
            waitFinishLatch.countDown();
        }
    });
    waitFinishLatch.await();
    cancelledCountDownLatch.await();
}
Also used : Action(org.apache.druid.server.security.Action) Access(org.apache.druid.server.security.Access) AuthConfig(org.apache.druid.server.security.AuthConfig) QueryInterruptedException(org.apache.druid.query.QueryInterruptedException) AuthenticationResult(org.apache.druid.server.security.AuthenticationResult) Authorizer(org.apache.druid.server.security.Authorizer) AuthorizerMapper(org.apache.druid.server.security.AuthorizerMapper) DefaultQueryConfig(org.apache.druid.query.DefaultQueryConfig) DefaultGenericQueryMetricsFactory(org.apache.druid.query.DefaultGenericQueryMetricsFactory) ObjectMapper(com.fasterxml.jackson.databind.ObjectMapper) DefaultObjectMapper(org.apache.druid.jackson.DefaultObjectMapper) Resource(org.apache.druid.server.security.Resource) NoopServiceEmitter(org.apache.druid.server.metrics.NoopServiceEmitter) IOException(java.io.IOException) CountDownLatch(java.util.concurrent.CountDownLatch) Response(javax.ws.rs.core.Response) ByteArrayInputStream(java.io.ByteArrayInputStream) DefaultObjectMapper(org.apache.druid.jackson.DefaultObjectMapper) Test(org.junit.Test)

Example 58 with DefaultObjectMapper

use of org.apache.druid.jackson.DefaultObjectMapper in project druid by druid-io.

the class QueryResourceTest method testDenySecuredCancelQuery.

@Test(timeout = 60_000L)
public void testDenySecuredCancelQuery() throws Exception {
    final CountDownLatch waitForCancellationLatch = new CountDownLatch(1);
    final CountDownLatch waitFinishLatch = new CountDownLatch(2);
    final CountDownLatch startAwaitLatch = new CountDownLatch(1);
    EasyMock.expect(testServletRequest.getAttribute(AuthConfig.DRUID_AUTHORIZATION_CHECKED)).andReturn(null).anyTimes();
    EasyMock.expect(testServletRequest.getAttribute(AuthConfig.DRUID_ALLOW_UNSECURED_PATH)).andReturn(null).anyTimes();
    EasyMock.expect(testServletRequest.getAttribute(AuthConfig.DRUID_AUTHENTICATION_RESULT)).andReturn(AUTHENTICATION_RESULT).anyTimes();
    testServletRequest.setAttribute(AuthConfig.DRUID_AUTHORIZATION_CHECKED, true);
    EasyMock.expectLastCall().times(1);
    testServletRequest.setAttribute(AuthConfig.DRUID_AUTHORIZATION_CHECKED, false);
    EasyMock.expectLastCall().times(1);
    EasyMock.replay(testServletRequest);
    AuthorizerMapper authMapper = new AuthorizerMapper(null) {

        @Override
        public Authorizer getAuthorizer(String name) {
            return new Authorizer() {

                @Override
                public Access authorize(AuthenticationResult authenticationResult, Resource resource, Action action) {
                    // WRITE corresponds to cancellation of query
                    if (action.equals(Action.READ)) {
                        try {
                            waitForCancellationLatch.await();
                        } catch (InterruptedException e) {
                            throw new RuntimeException(e);
                        }
                        return new Access(true);
                    } else {
                        // Deny access to cancel the query
                        return new Access(false);
                    }
                }
            };
        }
    };
    queryResource = new QueryResource(new QueryLifecycleFactory(WAREHOUSE, TEST_SEGMENT_WALKER, new DefaultGenericQueryMetricsFactory(), new NoopServiceEmitter(), testRequestLogger, new AuthConfig(), authMapper, Suppliers.ofInstance(new DefaultQueryConfig(ImmutableMap.of()))), jsonMapper, smileMapper, queryScheduler, new AuthConfig(), authMapper, ResponseContextConfig.newConfig(true), DRUID_NODE);
    final String queryString = "{\"queryType\":\"timeBoundary\", \"dataSource\":\"allow\"," + "\"context\":{\"queryId\":\"id_1\"}}";
    ObjectMapper mapper = new DefaultObjectMapper();
    Query<?> query = mapper.readValue(queryString, Query.class);
    ListenableFuture<?> future = MoreExecutors.listeningDecorator(Execs.singleThreaded("test_query_resource_%s")).submit(new Runnable() {

        @Override
        public void run() {
            try {
                startAwaitLatch.countDown();
                Response response = queryResource.doPost(new ByteArrayInputStream(queryString.getBytes(StandardCharsets.UTF_8)), null, testServletRequest);
                Assert.assertEquals(Response.Status.OK.getStatusCode(), response.getStatus());
            } catch (IOException e) {
                throw new RuntimeException(e);
            }
            waitFinishLatch.countDown();
        }
    });
    queryScheduler.registerQueryFuture(query, future);
    startAwaitLatch.await();
    Executors.newSingleThreadExecutor().submit(new Runnable() {

        @Override
        public void run() {
            try {
                queryResource.cancelQuery("id_1", testServletRequest);
            } catch (ForbiddenException e) {
                waitForCancellationLatch.countDown();
                waitFinishLatch.countDown();
            }
        }
    });
    waitFinishLatch.await();
}
Also used : Action(org.apache.druid.server.security.Action) Access(org.apache.druid.server.security.Access) AuthConfig(org.apache.druid.server.security.AuthConfig) QueryInterruptedException(org.apache.druid.query.QueryInterruptedException) AuthenticationResult(org.apache.druid.server.security.AuthenticationResult) Authorizer(org.apache.druid.server.security.Authorizer) AuthorizerMapper(org.apache.druid.server.security.AuthorizerMapper) DefaultQueryConfig(org.apache.druid.query.DefaultQueryConfig) DefaultGenericQueryMetricsFactory(org.apache.druid.query.DefaultGenericQueryMetricsFactory) ObjectMapper(com.fasterxml.jackson.databind.ObjectMapper) DefaultObjectMapper(org.apache.druid.jackson.DefaultObjectMapper) ForbiddenException(org.apache.druid.server.security.ForbiddenException) Resource(org.apache.druid.server.security.Resource) NoopServiceEmitter(org.apache.druid.server.metrics.NoopServiceEmitter) IOException(java.io.IOException) CountDownLatch(java.util.concurrent.CountDownLatch) Response(javax.ws.rs.core.Response) ByteArrayInputStream(java.io.ByteArrayInputStream) DefaultObjectMapper(org.apache.druid.jackson.DefaultObjectMapper) Test(org.junit.Test)

Example 59 with DefaultObjectMapper

use of org.apache.druid.jackson.DefaultObjectMapper in project druid by druid-io.

the class RequestLogLineTest method nullRemoteAddressAndNullSqlQueryContext.

@Test
public void nullRemoteAddressAndNullSqlQueryContext() throws JsonProcessingException {
    RequestLogLine requestLogLine = RequestLogLine.forNative(query, DateTimes.nowUtc(), null, new QueryStats(ImmutableMap.of()));
    Assert.assertEquals("", requestLogLine.getRemoteAddr());
    // call should not throw exception
    requestLogLine.getNativeQueryLine(new DefaultObjectMapper());
    requestLogLine = RequestLogLine.forSql("", null, DateTimes.nowUtc(), null, new QueryStats(ImmutableMap.of()));
    Assert.assertEquals("", requestLogLine.getRemoteAddr());
    Assert.assertEquals(ImmutableMap.<String, Object>of(), requestLogLine.getSqlQueryContext());
    // call should not throw exception
    requestLogLine.getSqlQueryLine(new DefaultObjectMapper());
}
Also used : DefaultObjectMapper(org.apache.druid.jackson.DefaultObjectMapper) Test(org.junit.Test)

Example 60 with DefaultObjectMapper

use of org.apache.druid.jackson.DefaultObjectMapper in project druid by druid-io.

the class ServerManagerTest method setUp.

@Before
public void setUp() {
    EmittingLogger.registerEmitter(new NoopServiceEmitter());
    queryWaitLatch = new CountDownLatch(1);
    queryWaitYieldLatch = new CountDownLatch(1);
    queryNotifyLatch = new CountDownLatch(1);
    factory = new MyQueryRunnerFactory(queryWaitLatch, queryWaitYieldLatch, queryNotifyLatch);
    serverManagerExec = Executors.newFixedThreadPool(2);
    segmentManager = new SegmentManager(new SegmentLoader() {

        @Override
        public ReferenceCountingSegment getSegment(final DataSegment segment, boolean lazy, SegmentLazyLoadFailCallback SegmentLazyLoadFailCallback) {
            return ReferenceCountingSegment.wrapSegment(new SegmentForTesting(MapUtils.getString(segment.getLoadSpec(), "version"), (Interval) segment.getLoadSpec().get("interval")), segment.getShardSpec());
        }

        @Override
        public void cleanup(DataSegment segment) {
        }
    });
    serverManager = new ServerManager(new QueryRunnerFactoryConglomerate() {

        @Override
        public <T, QueryType extends Query<T>> QueryRunnerFactory<T, QueryType> findFactory(QueryType query) {
            if (query instanceof SearchQuery) {
                return (QueryRunnerFactory) factory;
            } else {
                return null;
            }
        }
    }, new NoopServiceEmitter(), new ForwardingQueryProcessingPool(serverManagerExec), new ForegroundCachePopulator(new DefaultObjectMapper(), new CachePopulatorStats(), -1), new DefaultObjectMapper(), new LocalCacheProvider().get(), new CacheConfig(), segmentManager, NoopJoinableFactory.INSTANCE, new ServerConfig());
    loadQueryable("test", "1", Intervals.of("P1d/2011-04-01"));
    loadQueryable("test", "1", Intervals.of("P1d/2011-04-02"));
    loadQueryable("test", "2", Intervals.of("P1d/2011-04-02"));
    loadQueryable("test", "1", Intervals.of("P1d/2011-04-03"));
    loadQueryable("test", "1", Intervals.of("P1d/2011-04-04"));
    loadQueryable("test", "1", Intervals.of("P1d/2011-04-05"));
    loadQueryable("test", "2", Intervals.of("PT1h/2011-04-04T01"));
    loadQueryable("test", "2", Intervals.of("PT1h/2011-04-04T02"));
    loadQueryable("test", "2", Intervals.of("PT1h/2011-04-04T03"));
    loadQueryable("test", "2", Intervals.of("PT1h/2011-04-04T05"));
    loadQueryable("test", "2", Intervals.of("PT1h/2011-04-04T06"));
    loadQueryable("test2", "1", Intervals.of("P1d/2011-04-01"));
    loadQueryable("test2", "1", Intervals.of("P1d/2011-04-02"));
}
Also used : SearchQuery(org.apache.druid.query.search.SearchQuery) SegmentManager(org.apache.druid.server.SegmentManager) BaseQuery(org.apache.druid.query.BaseQuery) Query(org.apache.druid.query.Query) SearchQuery(org.apache.druid.query.search.SearchQuery) ForwardingQueryProcessingPool(org.apache.druid.query.ForwardingQueryProcessingPool) SegmentLazyLoadFailCallback(org.apache.druid.segment.SegmentLazyLoadFailCallback) NoopServiceEmitter(org.apache.druid.server.metrics.NoopServiceEmitter) CountDownLatch(java.util.concurrent.CountDownLatch) DataSegment(org.apache.druid.timeline.DataSegment) SegmentLoader(org.apache.druid.segment.loading.SegmentLoader) ServerConfig(org.apache.druid.server.initialization.ServerConfig) QueryRunnerFactoryConglomerate(org.apache.druid.query.QueryRunnerFactoryConglomerate) QueryRunnerFactory(org.apache.druid.query.QueryRunnerFactory) CachePopulatorStats(org.apache.druid.client.cache.CachePopulatorStats) DefaultObjectMapper(org.apache.druid.jackson.DefaultObjectMapper) LocalCacheProvider(org.apache.druid.client.cache.LocalCacheProvider) ForegroundCachePopulator(org.apache.druid.client.cache.ForegroundCachePopulator) CacheConfig(org.apache.druid.client.cache.CacheConfig) Interval(org.joda.time.Interval) Before(org.junit.Before)

Aggregations

DefaultObjectMapper (org.apache.druid.jackson.DefaultObjectMapper)287 Test (org.junit.Test)245 ObjectMapper (com.fasterxml.jackson.databind.ObjectMapper)177 File (java.io.File)26 InitializedNullHandlingTest (org.apache.druid.testing.InitializedNullHandlingTest)25 PostAggregator (org.apache.druid.query.aggregation.PostAggregator)21 Before (org.junit.Before)21 AggregatorFactory (org.apache.druid.query.aggregation.AggregatorFactory)20 InjectableValues (com.fasterxml.jackson.databind.InjectableValues)16 DataSegment (org.apache.druid.timeline.DataSegment)16 Period (org.joda.time.Period)16 ConstantPostAggregator (org.apache.druid.query.aggregation.post.ConstantPostAggregator)14 NamedType (com.fasterxml.jackson.databind.jsontype.NamedType)13 FieldAccessPostAggregator (org.apache.druid.query.aggregation.post.FieldAccessPostAggregator)12 DimensionsSpec (org.apache.druid.data.input.impl.DimensionsSpec)11 TaskStatus (org.apache.druid.indexer.TaskStatus)11 AvroStreamInputRowParserTest (org.apache.druid.data.input.AvroStreamInputRowParserTest)10 ImmutableList (com.google.common.collect.ImmutableList)9 Injector (com.google.inject.Injector)9 TimestampSpec (org.apache.druid.data.input.impl.TimestampSpec)9