Examples with Sequence org.apache.druid.java.util.common.guava.Sequence used on opensource projects

Example 86 with Sequence

use of org.apache.druid.java.util.common.guava.Sequence in project druid by druid-io.

the class IncrementalIndexStorageAdapterTest method testCursoringAndIndexUpdationInterleaving.

@Test
public void testCursoringAndIndexUpdationInterleaving() throws Exception {
    final IncrementalIndex index = indexCreator.createIndex();
    final long timestamp = System.currentTimeMillis();
    for (int i = 0; i < 2; i++) {
        index.add(new MapBasedInputRow(timestamp, Collections.singletonList("billy"), ImmutableMap.of("billy", "v1" + i)));
    }
    final StorageAdapter sa = new IncrementalIndexStorageAdapter(index);
    Sequence<Cursor> cursors = sa.makeCursors(null, Intervals.utc(timestamp - 60_000, timestamp + 60_000), VirtualColumns.EMPTY, Granularities.ALL, false, null);
    final AtomicInteger assertCursorsNotEmpty = new AtomicInteger(0);
    cursors.map(cursor -> {
        DimensionSelector dimSelector = cursor.getColumnSelectorFactory().makeDimensionSelector(new DefaultDimensionSpec("billy", "billy"));
        int cardinality = dimSelector.getValueCardinality();
        // index gets more rows at this point, while other thread is iterating over the cursor
        try {
            for (int i = 0; i < 1; i++) {
                index.add(new MapBasedInputRow(timestamp, Collections.singletonList("billy"), ImmutableMap.of("billy", "v2" + i)));
            }
        } catch (Exception ex) {
            throw new RuntimeException(ex);
        }
        int rowNumInCursor = 0;
        // and then, cursoring continues in the other thread
        while (!cursor.isDone()) {
            IndexedInts row = dimSelector.getRow();
            row.forEach(i -> Assert.assertTrue(i < cardinality));
            cursor.advance();
            rowNumInCursor++;
        }
        Assert.assertEquals(2, rowNumInCursor);
        assertCursorsNotEmpty.incrementAndGet();
        return null;
    }).toList();
    Assert.assertEquals(1, assertCursorsNotEmpty.get());
}

Example 87 with Sequence

use of org.apache.druid.java.util.common.guava.Sequence in project druid by druid-io.

the class IncrementalIndexStorageAdapterTest method testCursoringAndSnapshot.

@Test
public void testCursoringAndSnapshot() throws Exception {
    final IncrementalIndex index = indexCreator.createIndex();
    final long timestamp = System.currentTimeMillis();
    for (int i = 0; i < 2; i++) {
        index.add(new MapBasedInputRow(timestamp, Collections.singletonList("billy"), ImmutableMap.of("billy", "v0" + i)));
    }
    final StorageAdapter sa = new IncrementalIndexStorageAdapter(index);
    Sequence<Cursor> cursors = sa.makeCursors(null, Intervals.utc(timestamp - 60_000, timestamp + 60_000), VirtualColumns.EMPTY, Granularities.ALL, false, null);
    final AtomicInteger assertCursorsNotEmpty = new AtomicInteger(0);
    cursors.map(cursor -> {
        DimensionSelector dimSelector1A = cursor.getColumnSelectorFactory().makeDimensionSelector(new DefaultDimensionSpec("billy", "billy"));
        int cardinalityA = dimSelector1A.getValueCardinality();
        // index gets more rows at this point, while other thread is iterating over the cursor
        try {
            index.add(new MapBasedInputRow(timestamp, Collections.singletonList("billy"), ImmutableMap.of("billy", "v1")));
        } catch (Exception ex) {
            throw new RuntimeException(ex);
        }
        DimensionSelector dimSelector1B = cursor.getColumnSelectorFactory().makeDimensionSelector(new DefaultDimensionSpec("billy", "billy"));
        // index gets more rows at this point, while other thread is iterating over the cursor
        try {
            index.add(new MapBasedInputRow(timestamp, Collections.singletonList("billy"), ImmutableMap.of("billy", "v2")));
            index.add(new MapBasedInputRow(timestamp, Collections.singletonList("billy2"), ImmutableMap.of("billy2", "v3")));
        } catch (Exception ex) {
            throw new RuntimeException(ex);
        }
        DimensionSelector dimSelector1C = cursor.getColumnSelectorFactory().makeDimensionSelector(new DefaultDimensionSpec("billy", "billy"));
        DimensionSelector dimSelector2D = cursor.getColumnSelectorFactory().makeDimensionSelector(new DefaultDimensionSpec("billy2", "billy2"));
        // index gets more rows at this point, while other thread is iterating over the cursor
        try {
            index.add(new MapBasedInputRow(timestamp, Collections.singletonList("billy"), ImmutableMap.of("billy", "v3")));
            index.add(new MapBasedInputRow(timestamp, Collections.singletonList("billy3"), ImmutableMap.of("billy3", "")));
        } catch (Exception ex) {
            throw new RuntimeException(ex);
        }
        DimensionSelector dimSelector3E = cursor.getColumnSelectorFactory().makeDimensionSelector(new DefaultDimensionSpec("billy3", "billy3"));
        int rowNumInCursor = 0;
        // and then, cursoring continues in the other thread
        while (!cursor.isDone()) {
            IndexedInts rowA = dimSelector1A.getRow();
            rowA.forEach(i -> Assert.assertTrue(i < cardinalityA));
            IndexedInts rowB = dimSelector1B.getRow();
            rowB.forEach(i -> Assert.assertTrue(i < cardinalityA));
            IndexedInts rowC = dimSelector1C.getRow();
            rowC.forEach(i -> Assert.assertTrue(i < cardinalityA));
            IndexedInts rowD = dimSelector2D.getRow();
            // no null id, so should get empty dims array
            Assert.assertEquals(0, rowD.size());
            IndexedInts rowE = dimSelector3E.getRow();
            if (NullHandling.replaceWithDefault()) {
                Assert.assertEquals(1, rowE.size());
                // the null id
                Assert.assertEquals(0, rowE.get(0));
            } else {
                Assert.assertEquals(0, rowE.size());
            }
            cursor.advance();
            rowNumInCursor++;
        }
        Assert.assertEquals(2, rowNumInCursor);
        assertCursorsNotEmpty.incrementAndGet();
        return null;
    }).toList();
    Assert.assertEquals(1, assertCursorsNotEmpty.get());
}

Example 88 with Sequence

use of org.apache.druid.java.util.common.guava.Sequence in project druid by druid-io.

the class ChainedExecutionQueryRunner method run.

@Override
public Sequence<T> run(final QueryPlus<T> queryPlus, final ResponseContext responseContext) {
    Query<T> query = queryPlus.getQuery();
    final int priority = QueryContexts.getPriority(query);
    final Ordering ordering = query.getResultOrdering();
    final QueryPlus<T> threadSafeQueryPlus = queryPlus.withoutThreadUnsafeState();
    return new BaseSequence<T, Iterator<T>>(new BaseSequence.IteratorMaker<T, Iterator<T>>() {

        @Override
        public Iterator<T> make() {
            // Make it a List<> to materialize all of the values (so that it will submit everything to the executor)
            List<ListenableFuture<Iterable<T>>> futures = Lists.newArrayList(Iterables.transform(queryables, input -> {
                if (input == null) {
                    throw new ISE("Null queryRunner! Looks to be some segment unmapping action happening");
                }
                return queryProcessingPool.submitRunnerTask(new AbstractPrioritizedQueryRunnerCallable<Iterable<T>, T>(priority, input) {

                    @Override
                    public Iterable<T> call() {
                        try {
                            Sequence<T> result = input.run(threadSafeQueryPlus, responseContext);
                            if (result == null) {
                                throw new ISE("Got a null result! Segments are missing!");
                            }
                            List<T> retVal = result.toList();
                            if (retVal == null) {
                                throw new ISE("Got a null list of results");
                            }
                            return retVal;
                        } catch (QueryInterruptedException e) {
                            throw new RuntimeException(e);
                        } catch (QueryTimeoutException e) {
                            throw e;
                        } catch (Exception e) {
                            log.noStackTrace().error(e, "Exception with one of the sequences!");
                            Throwables.propagateIfPossible(e);
                            throw new RuntimeException(e);
                        }
                    }
                });
            }));
            ListenableFuture<List<Iterable<T>>> future = Futures.allAsList(futures);
            queryWatcher.registerQueryFuture(query, future);
            try {
                return new MergeIterable<>(ordering.nullsFirst(), QueryContexts.hasTimeout(query) ? future.get(QueryContexts.getTimeout(query), TimeUnit.MILLISECONDS) : future.get()).iterator();
            } catch (InterruptedException e) {
                log.noStackTrace().warn(e, "Query interrupted, cancelling pending results, query id [%s]", query.getId());
                // Note: canceling combinedFuture first so that it can complete with INTERRUPTED as its final state. See ChainedExecutionQueryRunnerTest.testQueryTimeout()
                GuavaUtils.cancelAll(true, future, futures);
                throw new QueryInterruptedException(e);
            } catch (CancellationException e) {
                throw new QueryInterruptedException(e);
            } catch (TimeoutException e) {
                log.warn("Query timeout, cancelling pending results for query id [%s]", query.getId());
                GuavaUtils.cancelAll(true, future, futures);
                throw new QueryTimeoutException(StringUtils.nonStrictFormat("Query [%s] timed out", query.getId()));
            } catch (ExecutionException e) {
                GuavaUtils.cancelAll(true, future, futures);
                Throwables.propagateIfPossible(e.getCause());
                throw new RuntimeException(e.getCause());
            }
        }

        @Override
        public void cleanup(Iterator<T> tIterator) {
        }
    });
}

Example 89 with Sequence

use of org.apache.druid.java.util.common.guava.Sequence in project druid by druid-io.

the class FinalizeResultsQueryRunner method run.

@Override
public Sequence<T> run(final QueryPlus<T> queryPlus, ResponseContext responseContext) {
    final Query<T> query = queryPlus.getQuery();
    final boolean isBySegment = QueryContexts.isBySegment(query);
    final boolean shouldFinalize = QueryContexts.isFinalize(query, true);
    final Query<T> queryToRun;
    final Function<T, ?> finalizerFn;
    final MetricManipulationFn metricManipulationFn;
    if (shouldFinalize) {
        queryToRun = query.withOverriddenContext(ImmutableMap.of("finalize", false));
        metricManipulationFn = MetricManipulatorFns.finalizing();
    } else {
        queryToRun = query;
        metricManipulationFn = MetricManipulatorFns.identity();
    }
    if (isBySegment) {
        finalizerFn = new Function<T, Result<BySegmentResultValue<T>>>() {

            final Function<T, T> baseFinalizer = toolChest.makePostComputeManipulatorFn(query, metricManipulationFn);

            @Override
            public Result<BySegmentResultValue<T>> apply(T input) {
                // noinspection unchecked (input is not actually a T; see class-level javadoc)
                Result<BySegmentResultValueClass<T>> result = (Result<BySegmentResultValueClass<T>>) input;
                if (input == null) {
                    throw new ISE("Cannot have a null result!");
                }
                BySegmentResultValue<T> resultsClass = result.getValue();
                return new Result<>(result.getTimestamp(), new BySegmentResultValueClass<>(Lists.transform(resultsClass.getResults(), baseFinalizer), resultsClass.getSegmentId(), resultsClass.getInterval()));
            }
        };
    } else {
        finalizerFn = toolChest.makePostComputeManipulatorFn(query, metricManipulationFn);
    }
    // noinspection unchecked (Technically unsound, but see class-level javadoc for rationale)
    return (Sequence<T>) Sequences.map(baseRunner.run(queryPlus.withQuery(queryToRun), responseContext), finalizerFn);
}

Example 90 with Sequence

use of org.apache.druid.java.util.common.guava.Sequence in project druid by druid-io.

the class UnionQueryRunner method run.

@Override
public Sequence<T> run(final QueryPlus<T> queryPlus, final ResponseContext responseContext) {
    Query<T> query = queryPlus.getQuery();
    final DataSourceAnalysis analysis = DataSourceAnalysis.forDataSource(query.getDataSource());
    if (analysis.isConcreteTableBased() && analysis.getBaseUnionDataSource().isPresent()) {
        // Union of tables.
        final UnionDataSource unionDataSource = analysis.getBaseUnionDataSource().get();
        if (unionDataSource.getDataSources().isEmpty()) {
            // Shouldn't happen, because UnionDataSource doesn't allow empty unions.
            throw new ISE("Unexpectedly received empty union");
        } else if (unionDataSource.getDataSources().size() == 1) {
            // Single table. Run as a normal query.
            return baseRunner.run(queryPlus.withQuery(Queries.withBaseDataSource(query, Iterables.getOnlyElement(unionDataSource.getDataSources()))), responseContext);
        } else {
            // Split up the tables and merge their results.
            return new MergeSequence<>(query.getResultOrdering(), Sequences.simple(Lists.transform(IntStream.range(0, unionDataSource.getDataSources().size()).mapToObj(i -> new Pair<>(unionDataSource.getDataSources().get(i), i + 1)).collect(Collectors.toList()), (Function<Pair<TableDataSource, Integer>, Sequence<T>>) singleSourceWithIndex -> baseRunner.run(queryPlus.withQuery(Queries.withBaseDataSource(query, singleSourceWithIndex.lhs).withSubQueryId(generateSubqueryId(query.getSubQueryId(), singleSourceWithIndex.lhs.getName(), singleSourceWithIndex.rhs))), responseContext))));
        }
    } else {
        // Not a union of tables. Do nothing special.
        return baseRunner.run(queryPlus, responseContext);
    }
}