Examples with ISE org.apache.druid.java.util.common.ISE used on opensource projects

Example 61 with ISE

use of org.apache.druid.java.util.common.ISE in project druid by druid-io.

the class GroupByMergingQueryRunnerV2 method run.

@Override
public Sequence<ResultRow> run(final QueryPlus<ResultRow> queryPlus, final ResponseContext responseContext) {
    final GroupByQuery query = (GroupByQuery) queryPlus.getQuery();
    final GroupByQueryConfig querySpecificConfig = config.withOverrides(query);
    // CTX_KEY_MERGE_RUNNERS_USING_CHAINED_EXECUTION is here because realtime servers use nested mergeRunners calls
    // (one for the entire query and one for each sink). We only want the outer call to actually do merging with a
    // merge buffer, otherwise the query will allocate too many merge buffers. This is potentially sub-optimal as it
    // will involve materializing the results for each sink before starting to feed them into the outer merge buffer.
    // I'm not sure of a better way to do this without tweaking how realtime servers do queries.
    final boolean forceChainedExecution = query.getContextBoolean(CTX_KEY_MERGE_RUNNERS_USING_CHAINED_EXECUTION, false);
    final QueryPlus<ResultRow> queryPlusForRunners = queryPlus.withQuery(query.withOverriddenContext(ImmutableMap.of(CTX_KEY_MERGE_RUNNERS_USING_CHAINED_EXECUTION, true))).withoutThreadUnsafeState();
    if (QueryContexts.isBySegment(query) || forceChainedExecution) {
        ChainedExecutionQueryRunner<ResultRow> runner = new ChainedExecutionQueryRunner<>(queryProcessingPool, queryWatcher, queryables);
        return runner.run(queryPlusForRunners, responseContext);
    }
    final boolean isSingleThreaded = querySpecificConfig.isSingleThreaded();
    final File temporaryStorageDirectory = new File(processingTmpDir, StringUtils.format("druid-groupBy-%s_%s", UUID.randomUUID(), query.getId()));
    final int priority = QueryContexts.getPriority(query);
    // Figure out timeoutAt time now, so we can apply the timeout to both the mergeBufferPool.take and the actual
    // query processing together.
    final long queryTimeout = QueryContexts.getTimeout(query);
    final boolean hasTimeout = QueryContexts.hasTimeout(query);
    final long timeoutAt = System.currentTimeMillis() + queryTimeout;
    return new BaseSequence<>(new BaseSequence.IteratorMaker<ResultRow, CloseableGrouperIterator<RowBasedKey, ResultRow>>() {

        @Override
        public CloseableGrouperIterator<RowBasedKey, ResultRow> make() {
            final Closer resources = Closer.create();
            try {
                final LimitedTemporaryStorage temporaryStorage = new LimitedTemporaryStorage(temporaryStorageDirectory, querySpecificConfig.getMaxOnDiskStorage());
                final ReferenceCountingResourceHolder<LimitedTemporaryStorage> temporaryStorageHolder = ReferenceCountingResourceHolder.fromCloseable(temporaryStorage);
                resources.register(temporaryStorageHolder);
                // If parallelCombine is enabled, we need two merge buffers for parallel aggregating and parallel combining
                final int numMergeBuffers = querySpecificConfig.getNumParallelCombineThreads() > 1 ? 2 : 1;
                final List<ReferenceCountingResourceHolder<ByteBuffer>> mergeBufferHolders = getMergeBuffersHolder(numMergeBuffers, hasTimeout, timeoutAt);
                resources.registerAll(mergeBufferHolders);
                final ReferenceCountingResourceHolder<ByteBuffer> mergeBufferHolder = mergeBufferHolders.get(0);
                final ReferenceCountingResourceHolder<ByteBuffer> combineBufferHolder = numMergeBuffers == 2 ? mergeBufferHolders.get(1) : null;
                Pair<Grouper<RowBasedKey>, Accumulator<AggregateResult, ResultRow>> pair = RowBasedGrouperHelper.createGrouperAccumulatorPair(query, null, config, Suppliers.ofInstance(mergeBufferHolder.get()), combineBufferHolder, concurrencyHint, temporaryStorage, spillMapper, // Passed as executor service
                queryProcessingPool, priority, hasTimeout, timeoutAt, mergeBufferSize);
                final Grouper<RowBasedKey> grouper = pair.lhs;
                final Accumulator<AggregateResult, ResultRow> accumulator = pair.rhs;
                grouper.init();
                final ReferenceCountingResourceHolder<Grouper<RowBasedKey>> grouperHolder = ReferenceCountingResourceHolder.fromCloseable(grouper);
                resources.register(grouperHolder);
                List<ListenableFuture<AggregateResult>> futures = Lists.newArrayList(Iterables.transform(queryables, new Function<QueryRunner<ResultRow>, ListenableFuture<AggregateResult>>() {

                    @Override
                    public ListenableFuture<AggregateResult> apply(final QueryRunner<ResultRow> input) {
                        if (input == null) {
                            throw new ISE("Null queryRunner! Looks to be some segment unmapping action happening");
                        }
                        ListenableFuture<AggregateResult> future = queryProcessingPool.submitRunnerTask(new AbstractPrioritizedQueryRunnerCallable<AggregateResult, ResultRow>(priority, input) {

                            @Override
                            public AggregateResult call() {
                                try (// These variables are used to close releasers automatically.
                                @SuppressWarnings("unused") Releaser bufferReleaser = mergeBufferHolder.increment();
                                    @SuppressWarnings("unused") Releaser grouperReleaser = grouperHolder.increment()) {
                                    // Return true if OK, false if resources were exhausted.
                                    return input.run(queryPlusForRunners, responseContext).accumulate(AggregateResult.ok(), accumulator);
                                } catch (QueryInterruptedException | QueryTimeoutException e) {
                                    throw e;
                                } catch (Exception e) {
                                    log.error(e, "Exception with one of the sequences!");
                                    throw new RuntimeException(e);
                                }
                            }
                        });
                        if (isSingleThreaded) {
                            waitForFutureCompletion(query, ImmutableList.of(future), hasTimeout, timeoutAt - System.currentTimeMillis());
                        }
                        return future;
                    }
                }));
                if (!isSingleThreaded) {
                    waitForFutureCompletion(query, futures, hasTimeout, timeoutAt - System.currentTimeMillis());
                }
                return RowBasedGrouperHelper.makeGrouperIterator(grouper, query, resources);
            } catch (Throwable t) {
                // Exception caught while setting up the iterator; release resources.
                try {
                    resources.close();
                } catch (Exception ex) {
                    t.addSuppressed(ex);
                }
                throw t;
            }
        }

        @Override
        public void cleanup(CloseableGrouperIterator<RowBasedKey, ResultRow> iterFromMake) {
            iterFromMake.close();
        }
    });
}

Example 62 with ISE

use of org.apache.druid.java.util.common.ISE in project druid by druid-io.

the class GroupByQueryHelper method createBySegmentAccumulatorPair.

public static <T> Pair<Queue, Accumulator<Queue, T>> createBySegmentAccumulatorPair() {
    // In parallel query runner multiple threads add to this queue concurrently
    Queue init = new ConcurrentLinkedQueue<>();
    Accumulator<Queue, T> accumulator = new Accumulator<Queue, T>() {

        @Override
        public Queue accumulate(Queue accumulated, T in) {
            if (in == null) {
                throw new ISE("Cannot have null result");
            }
            accumulated.offer(in);
            return accumulated;
        }
    };
    return new Pair<>(init, accumulator);
}

Example 63 with ISE

use of org.apache.druid.java.util.common.ISE in project druid by druid-io.

the class BufferArrayGrouper method init.

@Override
public void init() {
    if (!initialized) {
        final ByteBuffer buffer = bufferSupplier.get();
        final int usedFlagBufferEnd = Ints.checkedCast(getUsedFlagBufferCapacity(cardinalityWithMissingValue));
        // Sanity check on buffer capacity.
        if (usedFlagBufferEnd + (long) cardinalityWithMissingValue * recordSize > buffer.capacity()) {
            // enough scratch space.
            throw new ISE("Records of size[%,d] and possible cardinality[%,d] exceeds the buffer capacity[%,d].", recordSize, cardinalityWithMissingValue, valBuffer.capacity());
        }
        // Slice up the buffer.
        buffer.position(0);
        buffer.limit(usedFlagBufferEnd);
        usedFlagMemory = WritableMemory.writableWrap(buffer.slice(), ByteOrder.nativeOrder());
        buffer.position(usedFlagBufferEnd);
        buffer.limit(buffer.capacity());
        valBuffer = buffer.slice();
        reset();
        initialized = true;
    }
}

Example 64 with ISE

use of org.apache.druid.java.util.common.ISE in project druid by druid-io.

the class LimitedTemporaryStorage method createFile.

/**
 * Create a new temporary file. All methods of the returned output stream may throw
 * {@link TemporaryStorageFullException} if the temporary storage area fills up.
 *
 * @return output stream to the file
 *
 * @throws TemporaryStorageFullException if the temporary storage area is full
 * @throws IOException                   if something goes wrong while creating the file
 */
public LimitedOutputStream createFile() throws IOException {
    if (bytesUsed.get() >= maxBytesUsed) {
        throw new TemporaryStorageFullException(maxBytesUsed);
    }
    synchronized (files) {
        if (closed) {
            throw new ISE("Closed");
        }
        FileUtils.mkdirp(storageDirectory);
        if (!createdStorageDirectory) {
            createdStorageDirectory = true;
        }
        final File theFile = new File(storageDirectory, StringUtils.format("%08d.tmp", files.size()));
        final EnumSet<StandardOpenOption> openOptions = EnumSet.of(StandardOpenOption.CREATE_NEW, StandardOpenOption.WRITE);
        final FileChannel channel = FileChannel.open(theFile.toPath(), openOptions);
        files.add(theFile);
        return new LimitedOutputStream(theFile, Channels.newOutputStream(channel));
    }
}

Example 65 with ISE

use of org.apache.druid.java.util.common.ISE in project druid by druid-io.

the class ScanQueryRunnerTest method verify.

public static void verify(Iterable<ScanResultValue> expectedResults, Iterable<ScanResultValue> actualResults) {
    Iterator<ScanResultValue> expectedIter = expectedResults.iterator();
    Iterator<ScanResultValue> actualIter = actualResults.iterator();
    while (expectedIter.hasNext()) {
        ScanResultValue expected = expectedIter.next();
        ScanResultValue actual = actualIter.next();
        Assert.assertEquals(expected.getSegmentId(), actual.getSegmentId());
        Set exColumns = Sets.newTreeSet(expected.getColumns());
        Set acColumns = Sets.newTreeSet(actual.getColumns());
        Assert.assertEquals(exColumns, acColumns);
        Iterator<Map<String, Object>> expectedEvts = ((List<Map<String, Object>>) expected.getEvents()).iterator();
        Iterator<Map<String, Object>> actualEvts = ((List<Map<String, Object>>) actual.getEvents()).iterator();
        while (expectedEvts.hasNext()) {
            Map<String, Object> exHolder = expectedEvts.next();
            Map<String, Object> acHolder = actualEvts.next();
            for (Map.Entry<String, Object> ex : exHolder.entrySet()) {
                Object actVal = acHolder.get(ex.getKey());
                if (actVal instanceof String[]) {
                    actVal = Arrays.asList((String[]) actVal);
                }
                Object exValue = ex.getValue();
                if (exValue instanceof Double || exValue instanceof Float) {
                    final double expectedDoubleValue = ((Number) exValue).doubleValue();
                    Assert.assertEquals("invalid value for " + ex.getKey(), expectedDoubleValue, ((Number) actVal).doubleValue(), expectedDoubleValue * 1e-6);
                } else {
                    Assert.assertEquals("invalid value for " + ex.getKey(), ex.getValue(), actVal);
                }
            }
            for (Map.Entry<String, Object> ac : acHolder.entrySet()) {
                Object exVal = exHolder.get(ac.getKey());
                Object actVal = ac.getValue();
                if (actVal instanceof String[]) {
                    actVal = Arrays.asList((String[]) actVal);
                }
                if (exVal instanceof Double || exVal instanceof Float) {
                    final double exDoubleValue = ((Number) exVal).doubleValue();
                    Assert.assertEquals("invalid value for " + ac.getKey(), exDoubleValue, ((Number) actVal).doubleValue(), exDoubleValue * 1e-6);
                } else {
                    Assert.assertEquals("invalid value for " + ac.getKey(), exVal, actVal);
                }
            }
        }
        if (actualEvts.hasNext()) {
            throw new ISE("This event iterator should be exhausted!");
        }
    }
    if (actualIter.hasNext()) {
        throw new ISE("This iterator should be exhausted!");
    }
}