Examples with Sequence org.apache.druid.java.util.common.guava.Sequence used on opensource projects

Example 21 with Sequence

use of org.apache.druid.java.util.common.guava.Sequence in project druid by druid-io.

the class BackgroundCachePopulatorTest method testWrap.

/**
 * Method: wrap(final Sequence<T> sequence, final Function<T, CacheType> cacheFn, final Cache cache, final Cache.NamedKey cacheKey)
 */
@Test
public void testWrap() {
    String cacheId = "segment";
    SegmentDescriptor segmentDescriptor = new SegmentDescriptor(Intervals.of("2011/2012"), "version", 0);
    CacheStrategy cacheStrategy = toolchest.getCacheStrategy(query);
    Cache.NamedKey cacheKey = CacheUtil.computeSegmentCacheKey(cacheId, segmentDescriptor, cacheStrategy.computeCacheKey(query));
    Sequence res = this.backgroundCachePopulator.wrap(this.baseRunner.run(QueryPlus.wrap(query), ResponseContext.createEmpty()), (value) -> cacheStrategy.prepareForSegmentLevelCache().apply(value), cache, cacheKey);
    Assert.assertFalse("sequence must not be closed", closable.isClosed());
    Assert.assertNull("cache must be empty", cache.get(cacheKey));
    List results = res.toList();
    Assert.assertTrue(closable.isClosed());
    List<Result> expectedRes = makeTopNResults(false, OBJECTS);
    Assert.assertEquals(expectedRes.toString(), results.toString());
    Assert.assertEquals(5, results.size());
}

Example 22 with Sequence

use of org.apache.druid.java.util.common.guava.Sequence in project druid by druid-io.

the class BackgroundCachePopulatorTest method testWrapOnFailure.

@Test
public void testWrapOnFailure() {
    String cacheId = "segment";
    SegmentDescriptor segmentDescriptor = new SegmentDescriptor(Intervals.of("2011/2012"), "version", 0);
    CacheStrategy cacheStrategy = toolchest.getCacheStrategy(query);
    Cache.NamedKey cacheKey = CacheUtil.computeSegmentCacheKey(cacheId, segmentDescriptor, cacheStrategy.computeCacheKey(query));
    Sequence res = this.backgroundCachePopulator.wrap(this.baseRunner.run(QueryPlus.wrap(query), ResponseContext.createEmpty()), (value) -> {
        throw new RuntimeException("Error");
    }, cache, cacheKey);
    Assert.assertFalse("sequence must not be closed", closable.isClosed());
    Assert.assertNull("cache must be empty", cache.get(cacheKey));
    List results = res.toList();
    Assert.assertTrue(closable.isClosed());
    List<Result> expectedRes = makeTopNResults(false, OBJECTS);
    Assert.assertEquals(expectedRes.toString(), results.toString());
    Assert.assertEquals(5, results.size());
}

Example 23 with Sequence

use of org.apache.druid.java.util.common.guava.Sequence in project druid by druid-io.

the class TestHttpClient method go.

@Override
public <Intermediate, Final> ListenableFuture<Final> go(Request request, HttpResponseHandler<Intermediate, Final> handler, Duration readTimeout) {
    try {
        final Query query = objectMapper.readValue(request.getContent().array(), Query.class);
        final QueryRunner queryRunner = servers.get(request.getUrl()).getQueryRunner();
        if (queryRunner == null) {
            throw new ISE("Can't find queryRunner for url[%s]", request.getUrl());
        }
        final ResponseContext responseContext = ResponseContext.createEmpty();
        final Sequence sequence = queryRunner.run(QueryPlus.wrap(query), responseContext);
        final byte[] serializedContent;
        try (ByteArrayOutputStream baos = new ByteArrayOutputStream()) {
            objectMapper.writeValue(baos, sequence);
            serializedContent = baos.toByteArray();
        }
        final ResponseContext.SerializationResult serializationResult = responseContext.serializeWith(objectMapper, RESPONSE_CTX_HEADER_LEN_LIMIT);
        final HttpResponse response = new DefaultHttpResponse(HttpVersion.HTTP_1_1, HttpResponseStatus.OK);
        response.headers().add(QueryResource.HEADER_RESPONSE_CONTEXT, serializationResult.getResult());
        response.setContent(HeapChannelBufferFactory.getInstance().getBuffer(serializedContent, 0, serializedContent.length));
        final ClientResponse<Intermediate> intermClientResponse = handler.handleResponse(response, NOOP_TRAFFIC_COP);
        final ClientResponse<Final> finalClientResponse = handler.done(intermClientResponse);
        return Futures.immediateFuture(finalClientResponse.getObj());
    } catch (IOException e) {
        throw new RuntimeException(e);
    }
}

Example 24 with Sequence

use of org.apache.druid.java.util.common.guava.Sequence in project druid by druid-io.

the class DruidSchema method refreshSegmentsForDataSource.

/**
 * Attempt to refresh "segmentSignatures" for a set of segments for a particular dataSource. Returns the set of
 * segments actually refreshed, which may be a subset of the asked-for set.
 */
private Set<SegmentId> refreshSegmentsForDataSource(final String dataSource, final Set<SegmentId> segments) throws IOException {
    if (!segments.stream().allMatch(segmentId -> segmentId.getDataSource().equals(dataSource))) {
        // Sanity check. We definitely expect this to pass.
        throw new ISE("'segments' must all match 'dataSource'!");
    }
    log.debug("Refreshing metadata for dataSource[%s].", dataSource);
    final long startTime = System.currentTimeMillis();
    // Segment id string -> SegmentId object.
    final Map<String, SegmentId> segmentIdMap = Maps.uniqueIndex(segments, SegmentId::toString);
    final Set<SegmentId> retVal = new HashSet<>();
    final Sequence<SegmentAnalysis> sequence = runSegmentMetadataQuery(Iterables.limit(segments, MAX_SEGMENTS_PER_QUERY));
    Yielder<SegmentAnalysis> yielder = Yielders.each(sequence);
    try {
        while (!yielder.isDone()) {
            final SegmentAnalysis analysis = yielder.get();
            final SegmentId segmentId = segmentIdMap.get(analysis.getId());
            if (segmentId == null) {
                log.warn("Got analysis for segment[%s] we didn't ask for, ignoring.", analysis.getId());
            } else {
                final RowSignature rowSignature = analysisToRowSignature(analysis);
                log.debug("Segment[%s] has signature[%s].", segmentId, rowSignature);
                segmentMetadataInfo.compute(dataSource, (datasourceKey, dataSourceSegments) -> {
                    if (dataSourceSegments == null) {
                        // Datasource may have been removed or become unavailable while this refresh was ongoing.
                        log.warn("No segment map found with datasource[%s], skipping refresh of segment[%s]", datasourceKey, segmentId);
                        return null;
                    } else {
                        dataSourceSegments.compute(segmentId, (segmentIdKey, segmentMetadata) -> {
                            if (segmentMetadata == null) {
                                log.warn("No segment[%s] found, skipping refresh", segmentId);
                                return null;
                            } else {
                                final AvailableSegmentMetadata updatedSegmentMetadata = AvailableSegmentMetadata.from(segmentMetadata).withRowSignature(rowSignature).withNumRows(analysis.getNumRows()).build();
                                retVal.add(segmentId);
                                return updatedSegmentMetadata;
                            }
                        });
                        if (dataSourceSegments.isEmpty()) {
                            return null;
                        } else {
                            return dataSourceSegments;
                        }
                    }
                });
            }
            yielder = yielder.next(null);
        }
    } finally {
        yielder.close();
    }
    log.debug("Refreshed metadata for dataSource[%s] in %,d ms (%d segments queried, %d segments left).", dataSource, System.currentTimeMillis() - startTime, retVal.size(), segments.size() - retVal.size());
    return retVal;
}

Example 25 with Sequence

use of org.apache.druid.java.util.common.guava.Sequence in project druid by druid-io.

the class NativeQueryMaker method runQuery.

@Override
public Sequence<Object[]> runQuery(final DruidQuery druidQuery) {
    final Query<?> query = druidQuery.getQuery();
    if (plannerContext.getPlannerConfig().isRequireTimeCondition() && !(druidQuery.getDataSource() instanceof InlineDataSource)) {
        if (Intervals.ONLY_ETERNITY.equals(findBaseDataSourceIntervals(query))) {
            throw new CannotBuildQueryException("requireTimeCondition is enabled, all queries must include a filter condition on the __time column");
        }
    }
    int numFilters = plannerContext.getPlannerConfig().getMaxNumericInFilters();
    // Instead of IN(v1,v2,v3) user should specify IN('v1','v2','v3')
    if (numFilters != PlannerConfig.NUM_FILTER_NOT_USED) {
        if (query.getFilter() instanceof OrDimFilter) {
            OrDimFilter orDimFilter = (OrDimFilter) query.getFilter();
            int numBoundFilters = 0;
            for (DimFilter filter : orDimFilter.getFields()) {
                numBoundFilters += filter instanceof BoundDimFilter ? 1 : 0;
            }
            if (numBoundFilters > numFilters) {
                String dimension = ((BoundDimFilter) (orDimFilter.getFields().get(0))).getDimension();
                throw new UOE(StringUtils.format("The number of values in the IN clause for [%s] in query exceeds configured maxNumericFilter limit of [%s] for INs. Cast [%s] values of IN clause to String", dimension, numFilters, orDimFilter.getFields().size()));
            }
        }
    }
    final List<String> rowOrder;
    if (query instanceof TimeseriesQuery && !druidQuery.getGrouping().getDimensions().isEmpty()) {
        // Hack for timeseries queries: when generating them, DruidQuery.toTimeseriesQuery translates a dimension
        // based on a timestamp_floor expression into a 'granularity'. This is not reflected in the druidQuery's
        // output row signature, so we have to account for it here.
        // TODO: We can remove this once https://github.com/apache/druid/issues/9974 is done.
        final String timeDimension = Iterables.getOnlyElement(druidQuery.getGrouping().getDimensions()).getOutputName();
        rowOrder = druidQuery.getOutputRowSignature().getColumnNames().stream().map(f -> timeDimension.equals(f) ? ColumnHolder.TIME_COLUMN_NAME : f).collect(Collectors.toList());
    } else {
        rowOrder = druidQuery.getOutputRowSignature().getColumnNames();
    }
    final List<SqlTypeName> columnTypes = druidQuery.getOutputRowType().getFieldList().stream().map(f -> f.getType().getSqlTypeName()).collect(Collectors.toList());
    return execute(query, mapColumnList(rowOrder, fieldMapping), mapColumnList(columnTypes, fieldMapping));
}