Examples with DataSegment io.druid.timeline.DataSegment used on opensource projects

Example 96 with DataSegment

use of io.druid.timeline.DataSegment in project druid by druid-io.

the class AppenderatorImpl method mergeAndPush.

/**
   * Merge segment, push to deep storage. Should only be used on segments that have been fully persisted. Must only
   * be run in the single-threaded pushExecutor.
   *
   * @param identifier sink identifier
   * @param sink       sink to push
   *
   * @return segment descriptor, or null if the sink is no longer valid
   */
private DataSegment mergeAndPush(final SegmentIdentifier identifier, final Sink sink) {
    // Bail out if this sink is null or otherwise not what we expect.
    if (sinks.get(identifier) != sink) {
        log.warn("Sink for segment[%s] no longer valid, bailing out of mergeAndPush.", identifier);
        return null;
    }
    // Use a descriptor file to indicate that pushing has completed.
    final File persistDir = computePersistDir(identifier);
    final File mergedTarget = new File(persistDir, "merged");
    final File descriptorFile = computeDescriptorFile(identifier);
    // Sanity checks
    for (FireHydrant hydrant : sink) {
        if (sink.isWritable()) {
            throw new ISE("WTF?! Expected sink to be no longer writable before mergeAndPush. Segment[%s].", identifier);
        }
        synchronized (hydrant) {
            if (!hydrant.hasSwapped()) {
                throw new ISE("WTF?! Expected sink to be fully persisted before mergeAndPush. Segment[%s].", identifier);
            }
        }
    }
    try {
        if (descriptorFile.exists()) {
            // Already pushed.
            log.info("Segment[%s] already pushed.", identifier);
            return objectMapper.readValue(descriptorFile, DataSegment.class);
        }
        log.info("Pushing merged index for segment[%s].", identifier);
        removeDirectory(mergedTarget);
        if (mergedTarget.exists()) {
            throw new ISE("Merged target[%s] exists after removing?!", mergedTarget);
        }
        List<QueryableIndex> indexes = Lists.newArrayList();
        for (FireHydrant fireHydrant : sink) {
            Segment segment = fireHydrant.getSegment();
            final QueryableIndex queryableIndex = segment.asQueryableIndex();
            log.info("Adding hydrant[%s]", fireHydrant);
            indexes.add(queryableIndex);
        }
        final File mergedFile;
        mergedFile = indexMerger.mergeQueryableIndex(indexes, schema.getGranularitySpec().isRollup(), schema.getAggregators(), mergedTarget, tuningConfig.getIndexSpec());
        QueryableIndex index = indexIO.loadIndex(mergedFile);
        DataSegment segment = dataSegmentPusher.push(mergedFile, sink.getSegment().withDimensions(Lists.newArrayList(index.getAvailableDimensions())));
        objectMapper.writeValue(descriptorFile, segment);
        log.info("Pushed merged index for segment[%s], descriptor is: %s", identifier, segment);
        return segment;
    } catch (Exception e) {
        metrics.incrementFailedHandoffs();
        log.warn(e, "Failed to push merged index for segment[%s].", identifier);
        throw Throwables.propagate(e);
    }
}

Example 97 with DataSegment

use of io.druid.timeline.DataSegment in project druid by druid-io.

the class IndexingServiceClient method mergeSegments.

public void mergeSegments(List<DataSegment> segments) {
    final Iterator<DataSegment> segmentsIter = segments.iterator();
    if (!segmentsIter.hasNext()) {
        return;
    }
    final String dataSource = segmentsIter.next().getDataSource();
    while (segmentsIter.hasNext()) {
        DataSegment next = segmentsIter.next();
        if (!dataSource.equals(next.getDataSource())) {
            throw new IAE("Cannot merge segments of different dataSources[%s] and [%s]", dataSource, next.getDataSource());
        }
    }
    runQuery(new ClientAppendQuery(dataSource, segments));
}

Example 98 with DataSegment

use of io.druid.timeline.DataSegment in project druid by druid-io.

the class ServerSelector method getCandidates.

public List<DruidServerMetadata> getCandidates(final int numCandidates) {
    List<DruidServerMetadata> result = Lists.newArrayList();
    synchronized (this) {
        final DataSegment target = segment.get();
        for (Map.Entry<Integer, Set<QueryableDruidServer>> entry : toPrioritizedServers().entrySet()) {
            Set<QueryableDruidServer> servers = entry.getValue();
            TreeMap<Integer, Set<QueryableDruidServer>> tieredMap = Maps.newTreeMap();
            while (!servers.isEmpty()) {
                // strategy.pick() removes entry
                tieredMap.put(entry.getKey(), servers);
                QueryableDruidServer server = strategy.pick(tieredMap, target);
                if (server == null) {
                    // regard this as any server in tieredMap is not appropriate
                    break;
                }
                result.add(server.getServer().getMetadata());
                if (numCandidates > 0 && result.size() >= numCandidates) {
                    return result;
                }
                servers.remove(server);
            }
        }
    }
    return result;
}

Example 99 with DataSegment

use of io.druid.timeline.DataSegment in project druid by druid-io.

the class LocalDataSegmentFinder method findSegments.

@Override
public Set<DataSegment> findSegments(String workingDirPath, boolean updateDescriptor) throws SegmentLoadingException {
    final Set<DataSegment> segments = Sets.newHashSet();
    final File workingDir = new File(workingDirPath);
    if (!workingDir.isDirectory()) {
        throw new SegmentLoadingException("Working directory [%s] didn't exist !?", workingDir);
    }
    recursiveSearchSegments(segments, workingDir, updateDescriptor);
    return segments;
}

Example 100 with DataSegment

use of io.druid.timeline.DataSegment in project druid by druid-io.

the class DruidDataSource method removePartition.

public DruidDataSource removePartition(String partitionName) {
    synchronized (lock) {
        DataSegment dataPart = partitionNames.remove(partitionName);
        if (dataPart == null) {
            return this;
        }
        segmentsHolder.remove(dataPart);
    }
    return this;
}