Examples with ISE org.apache.druid.java.util.common.ISE used on opensource projects

Example 76 with ISE

use of org.apache.druid.java.util.common.ISE in project druid by druid-io.

the class Sink method makeNewCurrIndex.

private FireHydrant makeNewCurrIndex(long minTimestamp, DataSchema schema) {
    final IncrementalIndexSchema indexSchema = new IncrementalIndexSchema.Builder().withMinTimestamp(minTimestamp).withTimestampSpec(schema.getTimestampSpec()).withQueryGranularity(schema.getGranularitySpec().getQueryGranularity()).withDimensionsSpec(schema.getDimensionsSpec()).withMetrics(schema.getAggregators()).withRollup(schema.getGranularitySpec().isRollup()).build();
    // Build the incremental-index according to the spec that was chosen by the user
    final IncrementalIndex newIndex = appendableIndexSpec.builder().setIndexSchema(indexSchema).setMaxRowCount(maxRowsInMemory).setMaxBytesInMemory(maxBytesInMemory).setUseMaxMemoryEstimates(useMaxMemoryEstimates).build();
    final FireHydrant old;
    synchronized (hydrantLock) {
        if (writable) {
            old = currHydrant;
            int newCount = 0;
            int numHydrants = hydrants.size();
            if (numHydrants > 0) {
                FireHydrant lastHydrant = hydrants.get(numHydrants - 1);
                newCount = lastHydrant.getCount() + 1;
                if (!indexSchema.getDimensionsSpec().hasCustomDimensions()) {
                    Map<String, ColumnCapabilities> oldCapabilities;
                    if (lastHydrant.hasSwapped()) {
                        oldCapabilities = new HashMap<>();
                        ReferenceCountingSegment segment = lastHydrant.getIncrementedSegment();
                        try {
                            QueryableIndex oldIndex = segment.asQueryableIndex();
                            for (String dim : oldIndex.getAvailableDimensions()) {
                                dimOrder.add(dim);
                                oldCapabilities.put(dim, oldIndex.getColumnHolder(dim).getCapabilities());
                            }
                        } finally {
                            segment.decrement();
                        }
                    } else {
                        IncrementalIndex oldIndex = lastHydrant.getIndex();
                        dimOrder.addAll(oldIndex.getDimensionOrder());
                        oldCapabilities = oldIndex.getColumnCapabilities();
                    }
                    newIndex.loadDimensionIterable(dimOrder, oldCapabilities);
                }
            }
            currHydrant = new FireHydrant(newIndex, newCount, getSegment().getId());
            if (old != null) {
                numRowsExcludingCurrIndex.addAndGet(old.getIndex().size());
            }
            hydrants.add(currHydrant);
        } else {
            // Oops, someone called finishWriting while we were making this new index.
            newIndex.close();
            throw new ISE("finishWriting() called during swap");
        }
    }
    return old;
}

Example 77 with ISE

use of org.apache.druid.java.util.common.ISE in project druid by druid-io.

the class QueryLifecycle method runSimple.

/**
 * For callers who have already authorized their query, and where simplicity is desired over flexibility. This method
 * does it all in one call. Logs and metrics are emitted when the Sequence is either fully iterated or throws an
 * exception.
 *
 * @param query                 the query
 * @param authenticationResult  authentication result indicating identity of the requester
 * @param authorizationResult   authorization result of requester
 *
 * @return results
 */
@SuppressWarnings("unchecked")
public <T> Sequence<T> runSimple(final Query<T> query, final AuthenticationResult authenticationResult, final Access authorizationResult) {
    initialize(query);
    final Sequence<T> results;
    try {
        preAuthorized(authenticationResult, authorizationResult);
        if (!authorizationResult.isAllowed()) {
            throw new ISE("Unauthorized");
        }
        final QueryLifecycle.QueryResponse queryResponse = execute();
        results = queryResponse.getResults();
    } catch (Throwable e) {
        emitLogsAndMetrics(e, null, -1);
        throw e;
    }
    return Sequences.wrap(results, new SequenceWrapper() {

        @Override
        public void after(final boolean isDone, final Throwable thrown) {
            emitLogsAndMetrics(thrown, null, -1);
        }
    });
}

Example 78 with ISE

use of org.apache.druid.java.util.common.ISE in project druid by druid-io.

the class PrefetchSqlFirehoseFactory method connect.

@Override
public Firehose connect(InputRowParser<Map<String, Object>> firehoseParser, @Nullable File temporaryDirectory) {
    if (objects == null) {
        objects = ImmutableList.copyOf(Preconditions.checkNotNull(initObjects(), "objects"));
    }
    if (cacheManager.isEnabled() || fetchConfig.getMaxFetchCapacityBytes() > 0) {
        Preconditions.checkNotNull(temporaryDirectory, "temporaryDirectory");
        Preconditions.checkArgument(temporaryDirectory.exists(), "temporaryDirectory[%s] does not exist", temporaryDirectory);
        Preconditions.checkArgument(temporaryDirectory.isDirectory(), "temporaryDirectory[%s] is not a directory", temporaryDirectory);
    }
    LOG.info("Create a new firehose for [%d] queries", objects.size());
    // fetchExecutor is responsible for background data fetching
    final ExecutorService fetchExecutor = Execs.singleThreaded("firehose_fetch_%d");
    final Fetcher<T> fetcher = new SqlFetcher<>(cacheManager, objects, fetchExecutor, temporaryDirectory, fetchConfig, new ObjectOpenFunction<T>() {

        @Override
        public InputStream open(T object, File outFile) throws IOException {
            return openObjectStream(object, outFile);
        }

        @Override
        public InputStream open(T object) throws IOException {
            final File outFile = File.createTempFile("sqlresults_", null, temporaryDirectory);
            return openObjectStream(object, outFile);
        }
    });
    return new SqlFirehose(new Iterator<JsonIterator<Map<String, Object>>>() {

        @Override
        public boolean hasNext() {
            return fetcher.hasNext();
        }

        @Override
        public JsonIterator<Map<String, Object>> next() {
            if (!hasNext()) {
                throw new NoSuchElementException();
            }
            try {
                TypeReference<Map<String, Object>> type = new TypeReference<Map<String, Object>>() {
                };
                final OpenObject<T> openObject = fetcher.next();
                final InputStream stream = openObject.getObjectStream();
                return new JsonIterator<>(type, stream, openObject.getResourceCloser(), objectMapper);
            } catch (Exception ioe) {
                throw new RuntimeException(ioe);
            }
        }
    }, firehoseParser, () -> {
        fetchExecutor.shutdownNow();
        try {
            Preconditions.checkState(fetchExecutor.awaitTermination(fetchConfig.getFetchTimeout(), TimeUnit.MILLISECONDS));
        } catch (InterruptedException e) {
            Thread.currentThread().interrupt();
            throw new ISE("Failed to shutdown fetch executor during close");
        }
    });
}

Example 79 with ISE

use of org.apache.druid.java.util.common.ISE in project druid by druid-io.

the class SegmentPublisherHelper method annotateShardSpec.

/**
 * This method fills missing information in the shard spec if necessary when publishing segments.
 *
 * - When time chunk lock is used, the non-appending task should set the proper size of the core partitions for
 *   dynamically-partitioned segments. See {@link #annotateCorePartitionSetSizeFn}.
 * - When segment lock is used, the overwriting task should set the proper size of the atomic update group.
 *   See {@link #annotateAtomicUpdateGroupFn}.
 */
static Set<DataSegment> annotateShardSpec(Set<DataSegment> segments) {
    final Map<Interval, List<DataSegment>> intervalToSegments = new HashMap<>();
    segments.forEach(segment -> intervalToSegments.computeIfAbsent(segment.getInterval(), k -> new ArrayList<>()).add(segment));
    for (Entry<Interval, List<DataSegment>> entry : intervalToSegments.entrySet()) {
        final Interval interval = entry.getKey();
        final List<DataSegment> segmentsPerInterval = entry.getValue();
        final ShardSpec firstShardSpec = segmentsPerInterval.get(0).getShardSpec();
        final boolean anyMismatch = segmentsPerInterval.stream().anyMatch(segment -> segment.getShardSpec().getClass() != firstShardSpec.getClass());
        if (anyMismatch) {
            throw new ISE("Mismatched shardSpecs in interval[%s] for segments[%s]", interval, segmentsPerInterval);
        }
        final Function<DataSegment, DataSegment> annotateFn;
        if (firstShardSpec instanceof OverwriteShardSpec) {
            annotateFn = annotateAtomicUpdateGroupFn(segmentsPerInterval.size());
        } else if (firstShardSpec instanceof BuildingShardSpec) {
            // sanity check
            // BuildingShardSpec is used in non-appending mode. In this mode,
            // the segments in each interval should have contiguous partitionIds,
            // so that they can be queryable (see PartitionHolder.isComplete()).
            int expectedCorePartitionSetSize = segmentsPerInterval.size();
            int actualCorePartitionSetSize = Math.toIntExact(segmentsPerInterval.stream().filter(segment -> segment.getShardSpec().getPartitionNum() < expectedCorePartitionSetSize).count());
            if (expectedCorePartitionSetSize != actualCorePartitionSetSize) {
                LOG.errorSegments(segmentsPerInterval, "Cannot publish segments due to incomplete time chunk");
                throw new ISE("Cannot publish segments due to incomplete time chunk for interval[%s]. " + "Expected [%s] segments in the core partition, but only [%] segments are found. " + "See task logs for more details about these segments.", interval, expectedCorePartitionSetSize, actualCorePartitionSetSize);
            }
            annotateFn = annotateCorePartitionSetSizeFn(expectedCorePartitionSetSize);
        } else if (firstShardSpec instanceof BucketNumberedShardSpec) {
            throw new ISE("Cannot publish segments with shardSpec[%s]", firstShardSpec);
        } else {
            annotateFn = null;
        }
        if (annotateFn != null) {
            intervalToSegments.put(interval, segmentsPerInterval.stream().map(annotateFn).collect(Collectors.toList()));
        }
    }
    return intervalToSegments.values().stream().flatMap(Collection::stream).collect(Collectors.toSet());
}

Example 80 with ISE

use of org.apache.druid.java.util.common.ISE in project druid by druid-io.

the class BatchDataSegmentAnnouncer method announceSegment.

@Override
public void announceSegment(DataSegment segment) throws IOException {
    if (segmentLookup.containsKey(segment)) {
        log.info("Skipping announcement of segment [%s]. Announcement exists already.", segment.getId());
        return;
    }
    synchronized (lock) {
        if (segmentLookup.containsKey(segment)) {
            log.info("Skipping announcement of segment [%s]. Announcement exists already.", segment.getId());
            return;
        }
        DataSegment toAnnounce = segmentTransformer.apply(segment);
        changes.addChangeRequest(new SegmentChangeRequestLoad(toAnnounce));
        if (isSkipSegmentAnnouncementOnZk) {
            segmentLookup.put(segment, dummyZnode);
            return;
        }
        int newBytesLen = jsonMapper.writeValueAsBytes(toAnnounce).length;
        if (newBytesLen > config.getMaxBytesPerNode()) {
            throw new ISE("byte size %,d exceeds %,d", newBytesLen, config.getMaxBytesPerNode());
        }
        boolean done = false;
        if (!availableZNodes.isEmpty()) {
            // update existing batch
            Iterator<SegmentZNode> iter = availableZNodes.iterator();
            while (iter.hasNext() && !done) {
                SegmentZNode availableZNode = iter.next();
                if (availableZNode.getBytes().length + newBytesLen < config.getMaxBytesPerNode()) {
                    availableZNode.addSegment(toAnnounce);
                    log.info("Announcing segment[%s] at existing path[%s]", toAnnounce.getId(), availableZNode.getPath());
                    announcer.update(availableZNode.getPath(), availableZNode.getBytes());
                    segmentLookup.put(toAnnounce, availableZNode);
                    if (availableZNode.getCount() >= config.getSegmentsPerNode()) {
                        availableZNodes.remove(availableZNode);
                    }
                    done = true;
                } else {
                    // We could have kept the znode around for later use, however we remove it since segment announcements should
                    // have similar size unless there are significant schema changes. Removing the znode reduces the number of
                    // znodes that would be scanned at each announcement.
                    availableZNodes.remove(availableZNode);
                }
            }
        }
        if (!done) {
            assert (availableZNodes.isEmpty());
            // create new batch
            SegmentZNode availableZNode = new SegmentZNode(makeServedSegmentPath());
            availableZNode.addSegment(toAnnounce);
            log.info("Announcing segment[%s] at new path[%s]", toAnnounce.getId(), availableZNode.getPath());
            announcer.announce(availableZNode.getPath(), availableZNode.getBytes());
            segmentLookup.put(toAnnounce, availableZNode);
            availableZNodes.add(availableZNode);
        }
    }
}