Examples with SegmentAnalysis org.apache.druid.query.metadata.metadata.SegmentAnalysis used on opensource projects

Example 6 with SegmentAnalysis

use of org.apache.druid.query.metadata.metadata.SegmentAnalysis in project druid by druid-io.

the class SegmentMetadataQueryTest method testSegmentMetadataQueryWithDefaultAnalysisMerge.

private void testSegmentMetadataQueryWithDefaultAnalysisMerge(String column, ColumnAnalysis analysis) {
    SegmentAnalysis mergedSegmentAnalysis = new SegmentAnalysis(differentIds ? "merged" : SegmentId.dummy("testSegment").toString(), ImmutableList.of(expectedSegmentAnalysis1.getIntervals().get(0)), ImmutableMap.of("__time", new ColumnAnalysis(ColumnType.LONG, ValueType.LONG.toString(), false, false, 12090 * 2, null, null, null, null), "index", new ColumnAnalysis(ColumnType.DOUBLE, ValueType.DOUBLE.toString(), false, false, 9672 * 2, null, null, null, null), column, analysis), expectedSegmentAnalysis1.getSize() + expectedSegmentAnalysis2.getSize(), expectedSegmentAnalysis1.getNumRows() + expectedSegmentAnalysis2.getNumRows(), null, null, null, null);
    QueryToolChest toolChest = FACTORY.getToolchest();
    ExecutorService exec = Executors.newCachedThreadPool();
    QueryRunner myRunner = new FinalizeResultsQueryRunner<>(toolChest.mergeResults(FACTORY.mergeRunners(Execs.directExecutor(), Lists.newArrayList(toolChest.preMergeQueryDecoration(runner1), toolChest.preMergeQueryDecoration(runner2)))), toolChest);
    Query query = testQuery.withColumns(new ListColumnIncluderator(Arrays.asList("__time", "index", column)));
    TestHelper.assertExpectedObjects(ImmutableList.of(mergedSegmentAnalysis), myRunner.run(QueryPlus.wrap(query)), "failed SegmentMetadata merging query");
    exec.shutdownNow();
}

Example 7 with SegmentAnalysis

use of org.apache.druid.query.metadata.metadata.SegmentAnalysis in project druid by druid-io.

the class SegmentMetadataUnionQueryTest method testSegmentMetadataUnionQuery.

@Test
public void testSegmentMetadataUnionQuery() {
    SegmentAnalysis expected = new SegmentAnalysis(QueryRunnerTestHelper.SEGMENT_ID.toString(), Collections.singletonList(Intervals.of("2011-01-12T00:00:00.000Z/2011-04-15T00:00:00.001Z")), ImmutableMap.of("placement", new ColumnAnalysis(ColumnType.STRING, ValueType.STRING.toString(), false, false, mmap ? 43524 : 43056, 1, "preferred", "preferred", null)), mmap ? 800544 : 803324, 4836, null, null, null, null);
    SegmentMetadataQuery query = new Druids.SegmentMetadataQueryBuilder().dataSource(QueryRunnerTestHelper.UNION_DATA_SOURCE).intervals(QueryRunnerTestHelper.FULL_ON_INTERVAL_SPEC).toInclude(new ListColumnIncluderator(Collections.singletonList("placement"))).analysisTypes(SegmentMetadataQuery.AnalysisType.CARDINALITY, SegmentMetadataQuery.AnalysisType.SIZE, SegmentMetadataQuery.AnalysisType.INTERVAL, SegmentMetadataQuery.AnalysisType.MINMAX).build();
    List result = runner.run(QueryPlus.wrap(query)).toList();
    TestHelper.assertExpectedObjects(ImmutableList.of(expected), result, "failed SegmentMetadata union query");
}

Example 8 with SegmentAnalysis

use of org.apache.druid.query.metadata.metadata.SegmentAnalysis in project druid by druid-io.

the class DumpSegment method runMetadata.

private void runMetadata(final Injector injector, final QueryableIndex index) throws IOException {
    final ObjectMapper objectMapper = injector.getInstance(Key.get(ObjectMapper.class, Json.class)).copy().configure(JsonGenerator.Feature.AUTO_CLOSE_TARGET, false);
    final SegmentMetadataQuery query = new SegmentMetadataQuery(new TableDataSource("dataSource"), new SpecificSegmentSpec(new SegmentDescriptor(index.getDataInterval(), "0", 0)), new ListColumnIncluderator(getColumnsToInclude(index)), false, null, EnumSet.allOf(SegmentMetadataQuery.AnalysisType.class), false, false);
    withOutputStream(new Function<OutputStream, Object>() {

        @Override
        public Object apply(final OutputStream out) {
            evaluateSequenceForSideEffects(Sequences.map(executeQuery(injector, index, query), new Function<SegmentAnalysis, Object>() {

                @Override
                public Object apply(SegmentAnalysis analysis) {
                    try {
                        objectMapper.writeValue(out, analysis);
                    } catch (IOException e) {
                        throw new RuntimeException(e);
                    }
                    return null;
                }
            }));
            return null;
        }
    });
}

Example 9 with SegmentAnalysis

use of org.apache.druid.query.metadata.metadata.SegmentAnalysis in project druid by druid-io.

the class DruidSchema method refreshSegmentsForDataSource.

/**
 * Attempt to refresh "segmentSignatures" for a set of segments for a particular dataSource. Returns the set of
 * segments actually refreshed, which may be a subset of the asked-for set.
 */
private Set<SegmentId> refreshSegmentsForDataSource(final String dataSource, final Set<SegmentId> segments) throws IOException {
    if (!segments.stream().allMatch(segmentId -> segmentId.getDataSource().equals(dataSource))) {
        // Sanity check. We definitely expect this to pass.
        throw new ISE("'segments' must all match 'dataSource'!");
    }
    log.debug("Refreshing metadata for dataSource[%s].", dataSource);
    final long startTime = System.currentTimeMillis();
    // Segment id string -> SegmentId object.
    final Map<String, SegmentId> segmentIdMap = Maps.uniqueIndex(segments, SegmentId::toString);
    final Set<SegmentId> retVal = new HashSet<>();
    final Sequence<SegmentAnalysis> sequence = runSegmentMetadataQuery(Iterables.limit(segments, MAX_SEGMENTS_PER_QUERY));
    Yielder<SegmentAnalysis> yielder = Yielders.each(sequence);
    try {
        while (!yielder.isDone()) {
            final SegmentAnalysis analysis = yielder.get();
            final SegmentId segmentId = segmentIdMap.get(analysis.getId());
            if (segmentId == null) {
                log.warn("Got analysis for segment[%s] we didn't ask for, ignoring.", analysis.getId());
            } else {
                final RowSignature rowSignature = analysisToRowSignature(analysis);
                log.debug("Segment[%s] has signature[%s].", segmentId, rowSignature);
                segmentMetadataInfo.compute(dataSource, (datasourceKey, dataSourceSegments) -> {
                    if (dataSourceSegments == null) {
                        // Datasource may have been removed or become unavailable while this refresh was ongoing.
                        log.warn("No segment map found with datasource[%s], skipping refresh of segment[%s]", datasourceKey, segmentId);
                        return null;
                    } else {
                        dataSourceSegments.compute(segmentId, (segmentIdKey, segmentMetadata) -> {
                            if (segmentMetadata == null) {
                                log.warn("No segment[%s] found, skipping refresh", segmentId);
                                return null;
                            } else {
                                final AvailableSegmentMetadata updatedSegmentMetadata = AvailableSegmentMetadata.from(segmentMetadata).withRowSignature(rowSignature).withNumRows(analysis.getNumRows()).build();
                                retVal.add(segmentId);
                                return updatedSegmentMetadata;
                            }
                        });
                        if (dataSourceSegments.isEmpty()) {
                            return null;
                        } else {
                            return dataSourceSegments;
                        }
                    }
                });
            }
            yielder = yielder.next(null);
        }
    } finally {
        yielder.close();
    }
    log.debug("Refreshed metadata for dataSource[%s] in %,d ms (%d segments queried, %d segments left).", dataSource, System.currentTimeMillis() - startTime, retVal.size(), segments.size() - retVal.size());
    return retVal;
}

Example 10 with SegmentAnalysis

use of org.apache.druid.query.metadata.metadata.SegmentAnalysis in project druid by druid-io.

the class DruidSchemaTest method testSegmentMetadataColumnType.

@Test
public void testSegmentMetadataColumnType() {
    RowSignature signature = DruidSchema.analysisToRowSignature(new SegmentAnalysis("id", ImmutableList.of(Intervals.utc(1L, 2L)), ImmutableMap.of("a", new ColumnAnalysis(ColumnType.STRING, ColumnType.STRING.asTypeString(), false, true, 1234, 26, "a", "z", null), "count", new ColumnAnalysis(ColumnType.LONG, ColumnType.LONG.asTypeString(), false, true, 1234, 26, "a", "z", null)), 1234, 100, null, null, null, null));
    Assert.assertEquals(RowSignature.builder().add("a", ColumnType.STRING).add("count", ColumnType.LONG).build(), signature);
}