Examples with ISE org.apache.druid.java.util.common.ISE used on opensource projects

Example 1 with ISE

use of org.apache.druid.java.util.common.ISE in project druid by druid-io.

the class BaseFilterTest method selectCountUsingVectorizedFilteredAggregator.

private long selectCountUsingVectorizedFilteredAggregator(final DimFilter dimFilter) {
    Preconditions.checkState(makeFilter(dimFilter).canVectorizeMatcher(adapter), "Cannot vectorize filter: %s", dimFilter);
    try (final VectorCursor cursor = makeVectorCursor(null)) {
        final FilteredAggregatorFactory aggregatorFactory = new FilteredAggregatorFactory(new CountAggregatorFactory("count"), maybeOptimize(dimFilter));
        final VectorAggregator aggregator = aggregatorFactory.factorizeVector(cursor.getColumnSelectorFactory());
        final ByteBuffer buf = ByteBuffer.allocate(aggregatorFactory.getMaxIntermediateSizeWithNulls() * 2);
        // Use two slots: one for each form of aggregate.
        aggregator.init(buf, 0);
        aggregator.init(buf, aggregatorFactory.getMaxIntermediateSizeWithNulls());
        for (; !cursor.isDone(); cursor.advance()) {
            aggregator.aggregate(buf, 0, 0, cursor.getCurrentVectorSize());
            final int[] positions = new int[cursor.getCurrentVectorSize()];
            Arrays.fill(positions, aggregatorFactory.getMaxIntermediateSizeWithNulls());
            final int[] allRows = new int[cursor.getCurrentVectorSize()];
            for (int i = 0; i < allRows.length; i++) {
                allRows[i] = i;
            }
            aggregator.aggregate(buf, cursor.getCurrentVectorSize(), positions, allRows, 0);
        }
        final long val1 = (long) aggregator.get(buf, 0);
        final long val2 = (long) aggregator.get(buf, aggregatorFactory.getMaxIntermediateSizeWithNulls());
        if (val1 != val2) {
            throw new ISE("Oh no, val1[%d] != val2[%d]", val1, val2);
        }
        return val1;
    }
}

Example 2 with ISE

use of org.apache.druid.java.util.common.ISE in project druid by druid-io.

the class FireHydrant method swapSegment.

public void swapSegment(@Nullable Segment newSegment) {
    while (true) {
        ReferenceCountingSegment currentSegment = adapter.get();
        if (currentSegment == null && newSegment == null) {
            return;
        }
        if (currentSegment != null && newSegment != null && !newSegment.getId().equals(currentSegment.getId())) {
            // Sanity check: identifier should not change
            throw new ISE("Cannot swap identifier[%s] -> [%s]", currentSegment.getId(), newSegment.getId());
        }
        if (currentSegment == newSegment) {
            throw new ISE("Cannot swap to the same segment");
        }
        ReferenceCountingSegment newReferenceCountingSegment = newSegment != null ? ReferenceCountingSegment.wrapRootGenerationSegment(newSegment) : null;
        if (adapter.compareAndSet(currentSegment, newReferenceCountingSegment)) {
            if (currentSegment != null) {
                currentSegment.close();
            }
            index = null;
            return;
        }
    }
}

Example 3 with ISE

use of org.apache.druid.java.util.common.ISE in project druid by druid-io.

the class LookupReferencesManager method stop.

@LifecycleStop
public void stop() {
    if (!lifecycleLock.canStop()) {
        throw new ISE("can't stop.");
    }
    LOG.debug("LookupExtractorFactoryContainerProvider is stopping.");
    if (!testMode) {
        mainThread.interrupt();
        try {
            mainThread.join();
        } catch (InterruptedException ex) {
            throw new ISE("failed to stop, mainThread couldn't finish.");
        }
    }
    for (Map.Entry<String, LookupExtractorFactoryContainer> e : stateRef.get().lookupMap.entrySet()) {
        try {
            if (e.getValue().getLookupExtractorFactory().close()) {
                LOG.info("Closed lookup [%s].", e.getKey());
            } else {
                LOG.error("Failed to close lookup [%s].", e.getKey());
            }
        } catch (Exception ex) {
            LOG.error(ex, "Failed to close lookup [%s].", e.getKey());
        }
    }
    LOG.debug("LookupExtractorFactoryContainerProvider is stopped.");
}

Example 4 with ISE

use of org.apache.druid.java.util.common.ISE in project druid by druid-io.

the class NewestSegmentFirstIterator method findSegmentsToCompact.

/**
 * Find segments to compact together for the given intervalToSearch. It progressively searches the given
 * intervalToSearch in time order (latest first). The timeline lookup duration is one day. It means, the timeline is
 * looked up for the last one day of the given intervalToSearch, and the next day is searched again if the size of
 * found segments are not enough to compact. This is repeated until enough amount of segments are found.
 *
 * @return segments to compact
 */
private SegmentsToCompact findSegmentsToCompact(final String dataSourceName, final CompactibleTimelineObjectHolderCursor compactibleTimelineObjectHolderCursor, final DataSourceCompactionConfig config) {
    final long inputSegmentSize = config.getInputSegmentSizeBytes();
    while (compactibleTimelineObjectHolderCursor.hasNext()) {
        List<DataSegment> segments = compactibleTimelineObjectHolderCursor.next();
        final SegmentsToCompact candidates = new SegmentsToCompact(segments);
        if (!candidates.isEmpty()) {
            final boolean isCompactibleSize = candidates.getTotalSize() <= inputSegmentSize;
            final boolean needsCompaction = needsCompaction(config, candidates);
            if (isCompactibleSize && needsCompaction) {
                if (config.getGranularitySpec() != null && config.getGranularitySpec().getSegmentGranularity() != null) {
                    Interval interval = candidates.getUmbrellaInterval();
                    Set<Interval> intervalsCompacted = intervalCompactedForDatasource.computeIfAbsent(dataSourceName, k -> new HashSet<>());
                    // Skip this candidates if we have compacted the interval already
                    if (intervalsCompacted.contains(interval)) {
                        continue;
                    }
                    intervalsCompacted.add(interval);
                }
                return candidates;
            } else {
                if (!needsCompaction) {
                    // Collect statistic for segments that is already compacted
                    collectSegmentStatistics(compactedSegments, dataSourceName, candidates);
                } else {
                    // Collect statistic for segments that is skipped
                    // Note that if segments does not need compaction then we do not double count here
                    collectSegmentStatistics(skippedSegments, dataSourceName, candidates);
                    log.warn("total segment size[%d] for datasource[%s] and interval[%s] is larger than inputSegmentSize[%d]." + " Continue to the next interval.", candidates.getTotalSize(), candidates.segments.get(0).getDataSource(), candidates.segments.get(0).getInterval(), inputSegmentSize);
                }
            }
        } else {
            throw new ISE("No segment is found?");
        }
    }
    log.info("All segments look good! Nothing to compact");
    return new SegmentsToCompact();
}

Example 5 with ISE

use of org.apache.druid.java.util.common.ISE in project druid by druid-io.

the class CompactSegments method run.

@Override
public DruidCoordinatorRuntimeParams run(DruidCoordinatorRuntimeParams params) {
    LOG.info("Compact segments");
    final CoordinatorCompactionConfig dynamicConfig = params.getCoordinatorCompactionConfig();
    final CoordinatorStats stats = new CoordinatorStats();
    List<DataSourceCompactionConfig> compactionConfigList = dynamicConfig.getCompactionConfigs();
    if (dynamicConfig.getMaxCompactionTaskSlots() > 0) {
        Map<String, VersionedIntervalTimeline<String, DataSegment>> dataSources = params.getUsedSegmentsTimelinesPerDataSource();
        if (compactionConfigList != null && !compactionConfigList.isEmpty()) {
            Map<String, DataSourceCompactionConfig> compactionConfigs = compactionConfigList.stream().collect(Collectors.toMap(DataSourceCompactionConfig::getDataSource, Function.identity()));
            final List<TaskStatusPlus> compactionTasks = filterNonCompactionTasks(indexingServiceClient.getActiveTasks());
            // dataSource -> list of intervals for which compaction will be skipped in this run
            final Map<String, List<Interval>> intervalsToSkipCompaction = new HashMap<>();
            int numEstimatedNonCompleteCompactionTasks = 0;
            for (TaskStatusPlus status : compactionTasks) {
                final TaskPayloadResponse response = indexingServiceClient.getTaskPayload(status.getId());
                if (response == null) {
                    throw new ISE("Got a null paylord from overlord for task[%s]", status.getId());
                }
                if (COMPACTION_TASK_TYPE.equals(response.getPayload().getType())) {
                    final ClientCompactionTaskQuery compactionTaskQuery = (ClientCompactionTaskQuery) response.getPayload();
                    DataSourceCompactionConfig dataSourceCompactionConfig = compactionConfigs.get(status.getDataSource());
                    if (dataSourceCompactionConfig != null && dataSourceCompactionConfig.getGranularitySpec() != null) {
                        Granularity configuredSegmentGranularity = dataSourceCompactionConfig.getGranularitySpec().getSegmentGranularity();
                        if (configuredSegmentGranularity != null && compactionTaskQuery.getGranularitySpec() != null && !configuredSegmentGranularity.equals(compactionTaskQuery.getGranularitySpec().getSegmentGranularity())) {
                            // We will cancel active compaction task if segmentGranularity changes and we will need to
                            // re-compact the interval
                            LOG.info("Canceled task[%s] as task segmentGranularity is [%s] but compaction config " + "segmentGranularity is [%s]", status.getId(), compactionTaskQuery.getGranularitySpec().getSegmentGranularity(), configuredSegmentGranularity);
                            indexingServiceClient.cancelTask(status.getId());
                            continue;
                        }
                    }
                    // Skip interval as the current active compaction task is good
                    final Interval interval = compactionTaskQuery.getIoConfig().getInputSpec().getInterval();
                    intervalsToSkipCompaction.computeIfAbsent(status.getDataSource(), k -> new ArrayList<>()).add(interval);
                    // Since we keep the current active compaction task running, we count the active task slots
                    numEstimatedNonCompleteCompactionTasks += findMaxNumTaskSlotsUsedByOneCompactionTask(compactionTaskQuery.getTuningConfig());
                } else {
                    throw new ISE("task[%s] is not a compactionTask", status.getId());
                }
            }
            // Skip all the intervals locked by higher priority tasks for each datasource
            // This must be done after the invalid compaction tasks are cancelled
            // in the loop above so that their intervals are not considered locked
            getLockedIntervalsToSkip(compactionConfigList).forEach((dataSource, intervals) -> intervalsToSkipCompaction.computeIfAbsent(dataSource, ds -> new ArrayList<>()).addAll(intervals));
            final CompactionSegmentIterator iterator = policy.reset(compactionConfigs, dataSources, intervalsToSkipCompaction);
            int totalCapacity;
            if (dynamicConfig.isUseAutoScaleSlots()) {
                try {
                    totalCapacity = indexingServiceClient.getTotalWorkerCapacityWithAutoScale();
                } catch (Exception e) {
                    LOG.warn("Failed to get total worker capacity with auto scale slots. Falling back to current capacity count");
                    totalCapacity = indexingServiceClient.getTotalWorkerCapacity();
                }
            } else {
                totalCapacity = indexingServiceClient.getTotalWorkerCapacity();
            }
            final int compactionTaskCapacity = (int) Math.min(totalCapacity * dynamicConfig.getCompactionTaskSlotRatio(), dynamicConfig.getMaxCompactionTaskSlots());
            final int numAvailableCompactionTaskSlots;
            if (numEstimatedNonCompleteCompactionTasks > 0) {
                numAvailableCompactionTaskSlots = Math.max(0, compactionTaskCapacity - numEstimatedNonCompleteCompactionTasks);
            } else {
                // compactionTaskCapacity might be 0 if totalWorkerCapacity is low.
                // This guarantees that at least one slot is available if
                // compaction is enabled and numEstimatedNonCompleteCompactionTasks is 0.
                numAvailableCompactionTaskSlots = Math.max(1, compactionTaskCapacity);
            }
            LOG.info("Found [%d] available task slots for compaction out of [%d] max compaction task capacity", numAvailableCompactionTaskSlots, compactionTaskCapacity);
            stats.addToGlobalStat(AVAILABLE_COMPACTION_TASK_SLOT, numAvailableCompactionTaskSlots);
            stats.addToGlobalStat(MAX_COMPACTION_TASK_SLOT, compactionTaskCapacity);
            final Map<String, AutoCompactionSnapshot.Builder> currentRunAutoCompactionSnapshotBuilders = new HashMap<>();
            if (numAvailableCompactionTaskSlots > 0) {
                stats.accumulate(doRun(compactionConfigs, currentRunAutoCompactionSnapshotBuilders, numAvailableCompactionTaskSlots, iterator));
            } else {
                stats.accumulate(makeStats(currentRunAutoCompactionSnapshotBuilders, 0, iterator));
            }
        } else {
            LOG.info("compactionConfig is empty. Skip.");
            autoCompactionSnapshotPerDataSource.set(new HashMap<>());
        }
    } else {
        LOG.info("maxCompactionTaskSlots was set to 0. Skip compaction");
        autoCompactionSnapshotPerDataSource.set(new HashMap<>());
    }
    return params.buildFromExisting().withCoordinatorStats(stats).build();
}