Examples with IAE org.apache.druid.java.util.common.IAE used on opensource projects

Example 31 with IAE

use of org.apache.druid.java.util.common.IAE in project druid by druid-io.

the class SegmentAllocateAction method perform.

@Override
public SegmentIdWithShardSpec perform(final Task task, final TaskActionToolbox toolbox) {
    int attempt = 0;
    while (true) {
        attempt++;
        if (!task.getDataSource().equals(dataSource)) {
            throw new IAE("Task dataSource must match action dataSource, [%s] != [%s].", task.getDataSource(), dataSource);
        }
        final IndexerMetadataStorageCoordinator msc = toolbox.getIndexerMetadataStorageCoordinator();
        // 1) if something overlaps our timestamp, use that
        // 2) otherwise try preferredSegmentGranularity & going progressively smaller
        final Interval rowInterval = queryGranularity.bucket(timestamp).withChronology(ISOChronology.getInstanceUTC());
        final Set<DataSegment> usedSegmentsForRow = new HashSet<>(msc.retrieveUsedSegmentsForInterval(dataSource, rowInterval, Segments.ONLY_VISIBLE));
        final SegmentIdWithShardSpec identifier;
        if (usedSegmentsForRow.isEmpty()) {
            identifier = tryAllocateFirstSegment(toolbox, task, rowInterval);
        } else {
            identifier = tryAllocateSubsequentSegment(toolbox, task, rowInterval, usedSegmentsForRow.iterator().next());
        }
        if (identifier != null) {
            return identifier;
        }
        // Could not allocate a pending segment. There's a chance that this is because someone else inserted a segment
        // overlapping with this row between when we called "msc.retrieveUsedSegmentsForInterval" and now. Check it again,
        // and if it's different, repeat.
        Set<DataSegment> newUsedSegmentsForRow = new HashSet<>(msc.retrieveUsedSegmentsForInterval(dataSource, rowInterval, Segments.ONLY_VISIBLE));
        if (!newUsedSegmentsForRow.equals(usedSegmentsForRow)) {
            if (attempt < MAX_ATTEMPTS) {
                final long shortRandomSleep = 50 + (long) (ThreadLocalRandom.current().nextDouble() * 450);
                log.debug("Used segment set changed for rowInterval[%s]. Retrying segment allocation in %,dms (attempt = %,d).", rowInterval, shortRandomSleep, attempt);
                try {
                    Thread.sleep(shortRandomSleep);
                } catch (InterruptedException e) {
                    Thread.currentThread().interrupt();
                    throw new RuntimeException(e);
                }
            } else {
                log.error("Used segment set changed for rowInterval[%s]. Not trying again (attempt = %,d).", rowInterval, attempt);
                return null;
            }
        } else {
            return null;
        }
    }
}

Example 32 with IAE

use of org.apache.druid.java.util.common.IAE in project druid by druid-io.

the class SinglePhaseParallelIndexTaskRunner method findIntervalAndVersion.

private NonnullPair<Interval, String> findIntervalAndVersion(DateTime timestamp) throws IOException {
    final GranularitySpec granularitySpec = getIngestionSchema().getDataSchema().getGranularitySpec();
    // This method is called whenever subtasks need to allocate a new segment via the supervisor task.
    // As a result, this code is never called in the Overlord. For now using the materialized intervals
    // here is ok for performance reasons
    final Set<Interval> materializedBucketIntervals = granularitySpec.materializedBucketIntervals();
    // List locks whenever allocating a new segment because locks might be revoked and no longer valid.
    final List<TaskLock> locks = getToolbox().getTaskActionClient().submit(new LockListAction());
    final TaskLock revokedLock = locks.stream().filter(TaskLock::isRevoked).findAny().orElse(null);
    if (revokedLock != null) {
        throw new ISE("Lock revoked: [%s]", revokedLock);
    }
    final Map<Interval, String> versions = locks.stream().collect(Collectors.toMap(TaskLock::getInterval, TaskLock::getVersion));
    Interval interval;
    String version;
    if (!materializedBucketIntervals.isEmpty()) {
        // If granularity spec has explicit intervals, we just need to find the version associated to the interval.
        // This is because we should have gotten all required locks up front when the task starts up.
        final Optional<Interval> maybeInterval = granularitySpec.bucketInterval(timestamp);
        if (!maybeInterval.isPresent()) {
            throw new IAE("Could not find interval for timestamp [%s]", timestamp);
        }
        interval = maybeInterval.get();
        if (!materializedBucketIntervals.contains(interval)) {
            throw new ISE("Unspecified interval[%s] in granularitySpec[%s]", interval, granularitySpec);
        }
        version = ParallelIndexSupervisorTask.findVersion(versions, interval);
        if (version == null) {
            throw new ISE("Cannot find a version for interval[%s]", interval);
        }
    } else {
        // We don't have explicit intervals. We can use the segment granularity to figure out what
        // interval we need, but we might not have already locked it.
        interval = granularitySpec.getSegmentGranularity().bucket(timestamp);
        version = ParallelIndexSupervisorTask.findVersion(versions, interval);
        if (version == null) {
            final int maxAllowedLockCount = getIngestionSchema().getTuningConfig().getMaxAllowedLockCount();
            if (maxAllowedLockCount >= 0 && locks.size() >= maxAllowedLockCount) {
                throw new MaxAllowedLocksExceededException(maxAllowedLockCount);
            }
            // We don't have a lock for this interval, so we should lock it now.
            final TaskLock lock = Preconditions.checkNotNull(getToolbox().getTaskActionClient().submit(new TimeChunkLockTryAcquireAction(TaskLockType.EXCLUSIVE, interval)), "Cannot acquire a lock for interval[%s]", interval);
            if (lock.isRevoked()) {
                throw new ISE(StringUtils.format("Lock for interval [%s] was revoked.", interval));
            }
            version = lock.getVersion();
        }
    }
    return new NonnullPair<>(interval, version);
}

Example 33 with IAE

use of org.apache.druid.java.util.common.IAE in project druid by druid-io.

the class DruidInputSource method getTimelineForSegmentIds.

public static List<TimelineObjectHolder<String, DataSegment>> getTimelineForSegmentIds(CoordinatorClient coordinatorClient, String dataSource, List<WindowedSegmentId> segmentIds) {
    final SortedMap<Interval, TimelineObjectHolder<String, DataSegment>> timeline = new TreeMap<>(Comparators.intervalsByStartThenEnd());
    for (WindowedSegmentId windowedSegmentId : Preconditions.checkNotNull(segmentIds, "segmentIds")) {
        final DataSegment segment = coordinatorClient.fetchUsedSegment(dataSource, windowedSegmentId.getSegmentId());
        for (Interval interval : windowedSegmentId.getIntervals()) {
            final TimelineObjectHolder<String, DataSegment> existingHolder = timeline.get(interval);
            if (existingHolder != null) {
                if (!existingHolder.getVersion().equals(segment.getVersion())) {
                    throw new ISE("Timeline segments with the same interval should have the same version: " + "existing version[%s] vs new segment[%s]", existingHolder.getVersion(), segment);
                }
                existingHolder.getObject().add(segment.getShardSpec().createChunk(segment));
            } else {
                timeline.put(interval, new TimelineObjectHolder<>(interval, segment.getInterval(), segment.getVersion(), new PartitionHolder<>(segment.getShardSpec().createChunk(segment))));
            }
        }
    }
    // Validate that none of the given windows overlaps (except for when multiple segments share exactly the
    // same interval).
    Interval lastInterval = null;
    for (Interval interval : timeline.keySet()) {
        if (lastInterval != null && interval.overlaps(lastInterval)) {
            throw new IAE("Distinct intervals in input segments may not overlap: [%s] vs [%s]", lastInterval, interval);
        }
        lastInterval = interval;
    }
    return new ArrayList<>(timeline.values());
}

Example 34 with IAE

use of org.apache.druid.java.util.common.IAE in project druid by druid-io.

the class ArrayOfStringsNullSafeSerde method deserializeFromMemory.

@Override
public String[] deserializeFromMemory(final Memory mem, final int numItems) {
    final String[] array = new String[numItems];
    long offsetBytes = 0;
    for (int i = 0; i < numItems; i++) {
        // Read the length of the ith String
        Util.checkBounds(offsetBytes, Integer.BYTES, mem.getCapacity());
        final int strLength = mem.getInt(offsetBytes);
        offsetBytes += Integer.BYTES;
        if (strLength >= 0) {
            // Read the bytes for the String
            final byte[] bytes = new byte[strLength];
            Util.checkBounds(offsetBytes, strLength, mem.getCapacity());
            mem.getByteArray(offsetBytes, bytes, 0, strLength);
            offsetBytes += strLength;
            array[i] = new String(bytes, StandardCharsets.UTF_8);
        } else if (strLength != NULL_STRING_LENGTH) {
            throw new IAE("Illegal strLength [%s] at offset [%s]. Must be %s, 0 or a positive integer.", strLength, offsetBytes, NULL_STRING_LENGTH);
        }
    }
    return array;
}

Example 35 with IAE

use of org.apache.druid.java.util.common.IAE in project druid by druid-io.

the class SeekableStreamSupervisor method resetInternal.

@VisibleForTesting
public void resetInternal(DataSourceMetadata dataSourceMetadata) {
    if (dataSourceMetadata == null) {
        // Reset everything
        boolean result = indexerMetadataStorageCoordinator.deleteDataSourceMetadata(dataSource);
        log.info("Reset dataSource[%s] - dataSource metadata entry deleted? [%s]", dataSource, result);
        activelyReadingTaskGroups.values().forEach(group -> killTasksInGroup(group, "DataSourceMetadata is not found while reset"));
        activelyReadingTaskGroups.clear();
        partitionGroups.clear();
        partitionOffsets.clear();
    } else {
        if (!checkSourceMetadataMatch(dataSourceMetadata)) {
            throw new IAE("Datasource metadata instance does not match required, found instance of [%s]", dataSourceMetadata.getClass());
        }
        log.info("Reset dataSource[%s] with metadata[%s]", dataSource, dataSourceMetadata);
        // Reset only the partitions in dataSourceMetadata if it has not been reset yet
        @SuppressWarnings("unchecked") final SeekableStreamDataSourceMetadata<PartitionIdType, SequenceOffsetType> resetMetadata = (SeekableStreamDataSourceMetadata<PartitionIdType, SequenceOffsetType>) dataSourceMetadata;
        if (resetMetadata.getSeekableStreamSequenceNumbers().getStream().equals(ioConfig.getStream())) {
            // metadata can be null
            final DataSourceMetadata metadata = indexerMetadataStorageCoordinator.retrieveDataSourceMetadata(dataSource);
            if (metadata != null && !checkSourceMetadataMatch(metadata)) {
                throw new IAE("Datasource metadata instance does not match required, found instance of [%s]", metadata.getClass());
            }
            @SuppressWarnings("unchecked") final SeekableStreamDataSourceMetadata<PartitionIdType, SequenceOffsetType> currentMetadata = (SeekableStreamDataSourceMetadata<PartitionIdType, SequenceOffsetType>) metadata;
            // defend against consecutive reset requests from replicas
            // as well as the case where the metadata store do not have an entry for the reset partitions
            boolean doReset = false;
            for (Entry<PartitionIdType, SequenceOffsetType> resetPartitionOffset : resetMetadata.getSeekableStreamSequenceNumbers().getPartitionSequenceNumberMap().entrySet()) {
                final SequenceOffsetType partitionOffsetInMetadataStore = currentMetadata == null ? null : currentMetadata.getSeekableStreamSequenceNumbers().getPartitionSequenceNumberMap().get(resetPartitionOffset.getKey());
                final TaskGroup partitionTaskGroup = activelyReadingTaskGroups.get(getTaskGroupIdForPartition(resetPartitionOffset.getKey()));
                final boolean isSameOffset = partitionTaskGroup != null && partitionTaskGroup.startingSequences.get(resetPartitionOffset.getKey()).equals(resetPartitionOffset.getValue());
                if (partitionOffsetInMetadataStore != null || isSameOffset) {
                    doReset = true;
                    break;
                }
            }
            if (!doReset) {
                log.info("Ignoring duplicate reset request [%s]", dataSourceMetadata);
                return;
            }
            boolean metadataUpdateSuccess;
            if (currentMetadata == null) {
                metadataUpdateSuccess = true;
            } else {
                final DataSourceMetadata newMetadata = currentMetadata.minus(resetMetadata);
                try {
                    metadataUpdateSuccess = indexerMetadataStorageCoordinator.resetDataSourceMetadata(dataSource, newMetadata);
                } catch (IOException e) {
                    log.error("Resetting DataSourceMetadata failed [%s]", e.getMessage());
                    throw new RuntimeException(e);
                }
            }
            if (metadataUpdateSuccess) {
                resetMetadata.getSeekableStreamSequenceNumbers().getPartitionSequenceNumberMap().keySet().forEach(partition -> {
                    final int groupId = getTaskGroupIdForPartition(partition);
                    killTaskGroupForPartitions(ImmutableSet.of(partition), "DataSourceMetadata is updated while reset");
                    activelyReadingTaskGroups.remove(groupId);
                    // killTaskGroupForPartitions() cleans up partitionGroups.
                    // Add the removed groups back.
                    partitionGroups.computeIfAbsent(groupId, k -> new HashSet<>());
                    partitionOffsets.put(partition, getNotSetMarker());
                });
            } else {
                throw new ISE("Unable to reset metadata");
            }
        } else {
            log.warn("Reset metadata stream [%s] and supervisor's stream name [%s] do not match", resetMetadata.getSeekableStreamSequenceNumbers().getStream(), ioConfig.getStream());
        }
    }
}