Examples with CachePopulatorStats org.apache.druid.client.cache.CachePopulatorStats used on opensource projects

Example 1 with CachePopulatorStats

use of org.apache.druid.client.cache.CachePopulatorStats in project druid by druid-io.

the class QueryRunnerBasedOnClusteredClientTestBase method setupTestBase.

@Before
public void setupTestBase() {
    segmentGenerator = new SegmentGenerator();
    httpClient = new TestHttpClient(objectMapper);
    simpleServerView = new SimpleServerView(toolChestWarehouse, objectMapper, httpClient);
    cachingClusteredClient = new CachingClusteredClient(toolChestWarehouse, simpleServerView, MapCache.create(0), objectMapper, new ForegroundCachePopulator(objectMapper, new CachePopulatorStats(), 0), new CacheConfig(), new DruidHttpClientConfig(), QueryStackTests.getProcessingConfig(USE_PARALLEL_MERGE_POOL_CONFIGURED, DruidProcessingConfig.DEFAULT_NUM_MERGE_BUFFERS), ForkJoinPool.commonPool(), QueryStackTests.DEFAULT_NOOP_SCHEDULER, new MapJoinableFactory(ImmutableSet.of(), ImmutableMap.of()), new NoopServiceEmitter());
    servers = new ArrayList<>();
}

Example 2 with CachePopulatorStats

use of org.apache.druid.client.cache.CachePopulatorStats in project druid by druid-io.

the class CachingClusteredClientTest method setUp.

@Before
public void setUp() {
    timeline = new VersionedIntervalTimeline<>(Ordering.natural());
    serverView = EasyMock.createNiceMock(TimelineServerView.class);
    cache = MapCache.create(100000);
    client = makeClient(new ForegroundCachePopulator(JSON_MAPPER, new CachePopulatorStats(), -1));
    servers = new DruidServer[] { new DruidServer("test1", "test1", null, 10, ServerType.HISTORICAL, "bye", 0), new DruidServer("test2", "test2", null, 10, ServerType.HISTORICAL, "bye", 0), new DruidServer("test3", "test3", null, 10, ServerType.HISTORICAL, "bye", 0), new DruidServer("test4", "test4", null, 10, ServerType.HISTORICAL, "bye", 0), new DruidServer("test5", "test5", null, 10, ServerType.HISTORICAL, "bye", 0) };
}

Example 3 with CachePopulatorStats

use of org.apache.druid.client.cache.CachePopulatorStats in project druid by druid-io.

the class CachingClusteredClientTest method testCachingOverBulkLimitEnforcesLimit.

@Test
@SuppressWarnings("unchecked")
public void testCachingOverBulkLimitEnforcesLimit() {
    final int limit = 10;
    final Interval interval = Intervals.of("2011-01-01/2011-01-02");
    final TimeseriesQuery query = Druids.newTimeseriesQueryBuilder().dataSource(DATA_SOURCE).intervals(new MultipleIntervalSegmentSpec(ImmutableList.of(interval))).filters(DIM_FILTER).granularity(GRANULARITY).aggregators(AGGS).postAggregators(POST_AGGS).context(CONTEXT).randomQueryId().build();
    final ResponseContext context = initializeResponseContext();
    final Cache cache = EasyMock.createStrictMock(Cache.class);
    final Capture<Iterable<Cache.NamedKey>> cacheKeyCapture = EasyMock.newCapture();
    EasyMock.expect(cache.getBulk(EasyMock.capture(cacheKeyCapture))).andReturn(ImmutableMap.of()).once();
    EasyMock.replay(cache);
    client = makeClient(new ForegroundCachePopulator(JSON_MAPPER, new CachePopulatorStats(), -1), cache, limit);
    final DruidServer lastServer = servers[random.nextInt(servers.length)];
    final DataSegment dataSegment = EasyMock.createNiceMock(DataSegment.class);
    EasyMock.expect(dataSegment.getId()).andReturn(SegmentId.dummy(DATA_SOURCE)).anyTimes();
    EasyMock.replay(dataSegment);
    final ServerSelector selector = new ServerSelector(dataSegment, new HighestPriorityTierSelectorStrategy(new RandomServerSelectorStrategy()));
    selector.addServerAndUpdateSegment(new QueryableDruidServer(lastServer, null), dataSegment);
    timeline.add(interval, "v", new SingleElementPartitionChunk<>(selector));
    getDefaultQueryRunner().run(QueryPlus.wrap(query), context);
    Assert.assertTrue("Capture cache keys", cacheKeyCapture.hasCaptured());
    Assert.assertTrue("Cache key below limit", ImmutableList.copyOf(cacheKeyCapture.getValue()).size() <= limit);
    EasyMock.verify(cache);
    EasyMock.reset(cache);
    cacheKeyCapture.reset();
    EasyMock.expect(cache.getBulk(EasyMock.capture(cacheKeyCapture))).andReturn(ImmutableMap.of()).once();
    EasyMock.replay(cache);
    client = makeClient(new ForegroundCachePopulator(JSON_MAPPER, new CachePopulatorStats(), -1), cache, 0);
    getDefaultQueryRunner().run(QueryPlus.wrap(query), context);
    EasyMock.verify(cache);
    EasyMock.verify(dataSegment);
    Assert.assertTrue("Capture cache keys", cacheKeyCapture.hasCaptured());
    Assert.assertTrue("Cache Keys empty", ImmutableList.copyOf(cacheKeyCapture.getValue()).isEmpty());
}

Example 4 with CachePopulatorStats

use of org.apache.druid.client.cache.CachePopulatorStats in project druid by druid-io.

the class CachingClusteredClientTest method testOutOfOrderBackgroundCachePopulation.

@Test
public void testOutOfOrderBackgroundCachePopulation() {
    // to trigger the actual execution when we are ready to shuffle the order.
    abstract class DrainTask implements Runnable {
    }
    final ForwardingListeningExecutorService randomizingExecutorService = new ForwardingListeningExecutorService() {

        final ConcurrentLinkedDeque<Pair<SettableFuture, Object>> taskQueue = new ConcurrentLinkedDeque<>();

        final ListeningExecutorService delegate = MoreExecutors.listeningDecorator(// are complete before moving on to the next query run.
        Execs.directExecutor());

        @Override
        protected ListeningExecutorService delegate() {
            return delegate;
        }

        private <T> ListenableFuture<T> maybeSubmitTask(Object task, boolean wait) {
            if (wait) {
                SettableFuture<T> future = SettableFuture.create();
                taskQueue.addFirst(Pair.of(future, task));
                return future;
            } else {
                List<Pair<SettableFuture, Object>> tasks = Lists.newArrayList(taskQueue.iterator());
                Collections.shuffle(tasks, new Random(0));
                for (final Pair<SettableFuture, Object> pair : tasks) {
                    ListenableFuture future = pair.rhs instanceof Callable ? delegate.submit((Callable) pair.rhs) : delegate.submit((Runnable) pair.rhs);
                    Futures.addCallback(future, new FutureCallback() {

                        @Override
                        public void onSuccess(@Nullable Object result) {
                            pair.lhs.set(result);
                        }

                        @Override
                        public void onFailure(Throwable t) {
                            pair.lhs.setException(t);
                        }
                    });
                }
            }
            return task instanceof Callable ? delegate.submit((Callable) task) : (ListenableFuture<T>) delegate.submit((Runnable) task);
        }

        @SuppressWarnings("ParameterPackage")
        @Override
        public <T> ListenableFuture<T> submit(Callable<T> task) {
            return maybeSubmitTask(task, true);
        }

        @Override
        public ListenableFuture<?> submit(Runnable task) {
            if (task instanceof DrainTask) {
                return maybeSubmitTask(task, false);
            } else {
                return maybeSubmitTask(task, true);
            }
        }
    };
    client = makeClient(new BackgroundCachePopulator(randomizingExecutorService, JSON_MAPPER, new CachePopulatorStats(), -1));
    // callback to be run every time a query run is complete, to ensure all background
    // caching tasks are executed, and cache is populated before we move onto the next query
    queryCompletedCallback = new Runnable() {

        @Override
        public void run() {
            try {
                randomizingExecutorService.submit(new DrainTask() {

                    @Override
                    public void run() {
                    // no-op
                    }
                }).get();
            } catch (Exception e) {
                throw new RuntimeException(e);
            }
        }
    };
    final Druids.TimeseriesQueryBuilder builder = Druids.newTimeseriesQueryBuilder().dataSource(DATA_SOURCE).intervals(SEG_SPEC).filters(DIM_FILTER).granularity(GRANULARITY).aggregators(AGGS).postAggregators(POST_AGGS).context(CONTEXT).randomQueryId();
    QueryRunner runner = new FinalizeResultsQueryRunner(getDefaultQueryRunner(), new TimeseriesQueryQueryToolChest());
    testQueryCaching(runner, builder.build(), Intervals.of("2011-01-05/2011-01-10"), makeTimeResults(DateTimes.of("2011-01-05"), 85, 102, DateTimes.of("2011-01-06"), 412, 521, DateTimes.of("2011-01-07"), 122, 21894, DateTimes.of("2011-01-08"), 5, 20, DateTimes.of("2011-01-09"), 18, 521), Intervals.of("2011-01-10/2011-01-13"), makeTimeResults(DateTimes.of("2011-01-10"), 85, 102, DateTimes.of("2011-01-11"), 412, 521, DateTimes.of("2011-01-12"), 122, 21894));
}

Example 5 with CachePopulatorStats

use of org.apache.druid.client.cache.CachePopulatorStats in project druid by druid-io.

the class FireDepartmentTest method testSerde.

@Test
public void testSerde() throws Exception {
    ObjectMapper jsonMapper = new DefaultObjectMapper();
    jsonMapper.setInjectableValues(new InjectableValues.Std().addValue(ObjectMapper.class, jsonMapper));
    FireDepartment schema = new FireDepartment(new DataSchema("foo", jsonMapper.convertValue(new StringInputRowParser(new JSONParseSpec(new TimestampSpec("timestamp", "auto", null), new DimensionsSpec(DimensionsSpec.getDefaultSchemas(Arrays.asList("dim1", "dim2"))), null, null, null), null), Map.class), new AggregatorFactory[] { new CountAggregatorFactory("count") }, new UniformGranularitySpec(Granularities.HOUR, Granularities.MINUTE, null), null, jsonMapper), new RealtimeIOConfig(null, new RealtimePlumberSchool(null, null, null, null, null, null, null, NoopJoinableFactory.INSTANCE, TestHelper.getTestIndexMergerV9(OffHeapMemorySegmentWriteOutMediumFactory.instance()), TestHelper.getTestIndexIO(), MapCache.create(0), NO_CACHE_CONFIG, new CachePopulatorStats(), TestHelper.makeJsonMapper())), RealtimeTuningConfig.makeDefaultTuningConfig(new File("/tmp/nonexistent")));
    String json = jsonMapper.writeValueAsString(schema);
    FireDepartment newSchema = jsonMapper.readValue(json, FireDepartment.class);
    Assert.assertEquals(schema.getDataSchema().getDataSource(), newSchema.getDataSchema().getDataSource());
    Assert.assertEquals("/tmp/nonexistent", schema.getTuningConfig().getBasePersistDirectory().toString());
}