Examples with QueryableIndexSegment org.apache.druid.segment.QueryableIndexSegment used on opensource projects

Example 86 with QueryableIndexSegment

use of org.apache.druid.segment.QueryableIndexSegment in project druid by druid-io.

the class SearchQueryRunnerTest method testSearchWithNullValueInDimension.

@Test
public void testSearchWithNullValueInDimension() throws Exception {
    IncrementalIndex index = new OnheapIncrementalIndex.Builder().setIndexSchema(new IncrementalIndexSchema.Builder().withMinTimestamp(DateTimes.of("2011-01-12T00:00:00.000Z").getMillis()).build()).setMaxRowCount(10).build();
    index.add(new MapBasedInputRow(1481871600000L, Arrays.asList("name", "host"), ImmutableMap.of("name", "name1", "host", "host")));
    index.add(new MapBasedInputRow(1481871670000L, Arrays.asList("name", "table"), ImmutableMap.of("name", "name2", "table", "table")));
    SearchQuery searchQuery = Druids.newSearchQueryBuilder().dimensions(new DefaultDimensionSpec("table", "table")).dataSource(QueryRunnerTestHelper.DATA_SOURCE).granularity(QueryRunnerTestHelper.ALL_GRAN).intervals(QueryRunnerTestHelper.FULL_ON_INTERVAL_SPEC).context(ImmutableMap.of("searchStrategy", "cursorOnly")).build();
    QueryRunnerFactory factory = new SearchQueryRunnerFactory(SELECTOR, TOOL_CHEST, QueryRunnerTestHelper.NOOP_QUERYWATCHER);
    QueryRunner runner = factory.createRunner(new QueryableIndexSegment(TestIndex.persistRealtimeAndLoadMMapped(index), SegmentId.dummy("asdf")));
    List<SearchHit> expectedHits = new ArrayList<>();
    expectedHits.add(new SearchHit("table", "table", 1));
    expectedHits.add(new SearchHit("table", NullHandling.defaultStringValue(), 1));
    checkSearchQuery(searchQuery, runner, expectedHits);
}

Example 87 with QueryableIndexSegment

use of org.apache.druid.segment.QueryableIndexSegment in project druid by druid-io.

the class TopNQueryRunnerBenchmark method setUp.

@BeforeClass
public static void setUp() {
    QueryRunnerFactory factory = new TopNQueryRunnerFactory(new StupidPool<ByteBuffer>("TopNQueryRunnerFactory-directBufferPool", new Supplier<ByteBuffer>() {

        @Override
        public ByteBuffer get() {
            // Instead of causing a circular dependency, we simply mimic its behavior
            return ByteBuffer.allocateDirect(2000);
        }
    }), new TopNQueryQueryToolChest(new TopNQueryConfig()), QueryRunnerTestHelper.NOOP_QUERYWATCHER);
    TEST_CASE_MAP.put(TestCases.rtIndex, QueryRunnerTestHelper.makeQueryRunner(factory, new IncrementalIndexSegment(TestIndex.getIncrementalTestIndex(), SEGMENT_ID), null));
    TEST_CASE_MAP.put(TestCases.mMappedTestIndex, QueryRunnerTestHelper.makeQueryRunner(factory, new QueryableIndexSegment(TestIndex.getMMappedTestIndex(), SEGMENT_ID), null));
    TEST_CASE_MAP.put(TestCases.mergedRealtimeIndex, QueryRunnerTestHelper.makeQueryRunner(factory, new QueryableIndexSegment(TestIndex.mergedRealtimeIndex(), SEGMENT_ID), null));
// Thread.sleep(10000);
}

Example 88 with QueryableIndexSegment

use of org.apache.druid.segment.QueryableIndexSegment in project druid by druid-io.

the class SearchQueryRunnerWithCaseTest method constructorFeeder.

@Parameterized.Parameters
public static Iterable<Object[]> constructorFeeder() {
    final SearchQueryConfig[] configs = new SearchQueryConfig[3];
    configs[0] = new SearchQueryConfig();
    configs[0].setSearchStrategy(UseIndexesStrategy.NAME);
    configs[1] = new SearchQueryConfig();
    configs[1].setSearchStrategy(CursorOnlyStrategy.NAME);
    configs[2] = new SearchQueryConfig();
    configs[2].setSearchStrategy(AutoStrategy.NAME);
    CharSource input = CharSource.wrap("2011-01-12T00:00:00.000Z\tspot\tAutoMotive\t1000\t10000.0\t10000.0\t100000\t10\t10.0\t10.0\tPREFERRED\ta\u0001preferred\t100.000000\n" + "2011-01-12T00:00:00.000Z\tSPot\tbusiness\t1100\t11000.0\t11000.0\t110000\t20\t20.0\t20.0\tpreferred\tb\u0001Preferred\t100.000000\n" + "2011-01-12T00:00:00.000Z\tspot\tentertainment\t1200\t12000.0\t12000.0\t120000\t\t\t\tPREFERRed\te\u0001preferred\t100.000000\n" + "2011-01-13T00:00:00.000Z\tspot\tautomotive\t1000\t10000.0\t10000.0\t100000\t10\t10.0\t10.0\tpreferred\ta\u0001preferred\t94.874713");
    IncrementalIndex index1 = TestIndex.makeRealtimeIndex(input);
    IncrementalIndex index2 = TestIndex.makeRealtimeIndex(input);
    QueryableIndex index3 = TestIndex.persistRealtimeAndLoadMMapped(index1);
    QueryableIndex index4 = TestIndex.persistRealtimeAndLoadMMapped(index2);
    final List<QueryRunner<Result<SearchResultValue>>> runners = new ArrayList<>();
    for (SearchQueryConfig config : configs) {
        runners.addAll(Arrays.asList(QueryRunnerTestHelper.makeQueryRunner(makeRunnerFactory(config), SegmentId.dummy("index1"), new IncrementalIndexSegment(index1, SegmentId.dummy("index1")), "index1"), QueryRunnerTestHelper.makeQueryRunner(makeRunnerFactory(config), SegmentId.dummy("index2"), new IncrementalIndexSegment(index2, SegmentId.dummy("index2")), "index2"), QueryRunnerTestHelper.makeQueryRunner(makeRunnerFactory(config), SegmentId.dummy("index3"), new QueryableIndexSegment(index3, SegmentId.dummy("index3")), "index3"), QueryRunnerTestHelper.makeQueryRunner(makeRunnerFactory(config), SegmentId.dummy("index4"), new QueryableIndexSegment(index4, SegmentId.dummy("index4")), "index4")));
    }
    return QueryRunnerTestHelper.transformToConstructionFeeder(runners);
}

Example 89 with QueryableIndexSegment

use of org.apache.druid.segment.QueryableIndexSegment in project druid by druid-io.

the class RealtimePlumber method persistHydrant.

/**
 * Persists the given hydrant and returns the number of rows persisted
 *
 * @param indexToPersist hydrant to persist
 * @param schema         datasource schema
 * @param interval       interval to persist
 *
 * @return the number of rows persisted
 */
protected int persistHydrant(FireHydrant indexToPersist, DataSchema schema, Interval interval, Map<String, Object> metadataElems) {
    synchronized (indexToPersist) {
        if (indexToPersist.hasSwapped()) {
            log.info("DataSource[%s], Interval[%s], Hydrant[%s] already swapped. Ignoring request to persist.", schema.getDataSource(), interval, indexToPersist);
            return 0;
        }
        log.info("DataSource[%s], Interval[%s], Metadata [%s] persisting Hydrant[%s]", schema.getDataSource(), interval, metadataElems, indexToPersist);
        try {
            int numRows = indexToPersist.getIndex().size();
            indexToPersist.getIndex().getMetadata().putAll(metadataElems);
            final File persistedFile = indexMerger.persist(indexToPersist.getIndex(), interval, new File(computePersistDir(schema, interval), String.valueOf(indexToPersist.getCount())), config.getIndexSpecForIntermediatePersists(), config.getSegmentWriteOutMediumFactory());
            indexToPersist.swapSegment(new QueryableIndexSegment(indexIO.loadIndex(persistedFile), indexToPersist.getSegmentId()));
            return numRows;
        } catch (IOException e) {
            log.makeAlert("dataSource[%s] -- incremental persist failed", schema.getDataSource()).addData("interval", interval).addData("count", indexToPersist.getCount()).emit();
            throw new RuntimeException(e);
        }
    }
}

Example 90 with QueryableIndexSegment

use of org.apache.druid.segment.QueryableIndexSegment in project druid by druid-io.

the class RealtimePlumber method bootstrapSinksFromDisk.

protected Object bootstrapSinksFromDisk() {
    final VersioningPolicy versioningPolicy = config.getVersioningPolicy();
    File baseDir = computeBaseDir(schema);
    if (baseDir == null || !baseDir.exists()) {
        return null;
    }
    File[] files = baseDir.listFiles();
    if (files == null) {
        return null;
    }
    Object metadata = null;
    long latestCommitTime = 0;
    for (File sinkDir : files) {
        final Interval sinkInterval = Intervals.of(sinkDir.getName().replace('_', '/'));
        // final File[] sinkFiles = sinkDir.listFiles();
        // To avoid reading and listing of "merged" dir
        final File[] sinkFiles = sinkDir.listFiles(new FilenameFilter() {

            @Override
            public boolean accept(File dir, String fileName) {
                return !(Ints.tryParse(fileName) == null);
            }
        });
        Arrays.sort(sinkFiles, new Comparator<File>() {

            @Override
            public int compare(File o1, File o2) {
                try {
                    return Ints.compare(Integer.parseInt(o1.getName()), Integer.parseInt(o2.getName()));
                } catch (NumberFormatException e) {
                    log.error(e, "Couldn't compare as numbers? [%s][%s]", o1, o2);
                    return o1.compareTo(o2);
                }
            }
        });
        boolean isCorrupted = false;
        List<FireHydrant> hydrants = new ArrayList<>();
        for (File segmentDir : sinkFiles) {
            log.info("Loading previously persisted segment at [%s]", segmentDir);
            // If 100% sure that this is not needed, this check can be removed.
            if (Ints.tryParse(segmentDir.getName()) == null) {
                continue;
            }
            QueryableIndex queryableIndex = null;
            try {
                queryableIndex = indexIO.loadIndex(segmentDir);
            } catch (IOException e) {
                log.error(e, "Problem loading segmentDir from disk.");
                isCorrupted = true;
            }
            if (isCorrupted) {
                try {
                    File corruptSegmentDir = computeCorruptedFileDumpDir(segmentDir, schema);
                    log.info("Renaming %s to %s", segmentDir.getAbsolutePath(), corruptSegmentDir.getAbsolutePath());
                    org.apache.commons.io.FileUtils.copyDirectory(segmentDir, corruptSegmentDir);
                    FileUtils.deleteDirectory(segmentDir);
                } catch (Exception e1) {
                    log.error(e1, "Failed to rename %s", segmentDir.getAbsolutePath());
                }
                // at some point.
                continue;
            }
            Metadata segmentMetadata = queryableIndex.getMetadata();
            if (segmentMetadata != null) {
                Object timestampObj = segmentMetadata.get(COMMIT_METADATA_TIMESTAMP_KEY);
                if (timestampObj != null) {
                    long timestamp = ((Long) timestampObj).longValue();
                    if (timestamp > latestCommitTime) {
                        log.info("Found metaData [%s] with latestCommitTime [%s] greater than previous recorded [%s]", queryableIndex.getMetadata(), timestamp, latestCommitTime);
                        latestCommitTime = timestamp;
                        metadata = queryableIndex.getMetadata().get(COMMIT_METADATA_KEY);
                    }
                }
            }
            hydrants.add(new FireHydrant(new QueryableIndexSegment(queryableIndex, SegmentId.of(schema.getDataSource(), sinkInterval, versioningPolicy.getVersion(sinkInterval), config.getShardSpec())), Integer.parseInt(segmentDir.getName())));
        }
        if (hydrants.isEmpty()) {
            // Probably encountered a corrupt sink directory
            log.warn("Found persisted segment directory with no intermediate segments present at %s, skipping sink creation.", sinkDir.getAbsolutePath());
            continue;
        }
        final Sink currSink = new Sink(sinkInterval, schema, config.getShardSpec(), versioningPolicy.getVersion(sinkInterval), config.getAppendableIndexSpec(), config.getMaxRowsInMemory(), config.getMaxBytesInMemoryOrDefault(), true, config.getDedupColumn(), hydrants);
        addSink(currSink);
    }
    return metadata;
}