Examples with SegmentListUsedAction io.druid.indexing.common.actions.SegmentListUsedAction used on opensource projects

Example 6 with SegmentListUsedAction

use of io.druid.indexing.common.actions.SegmentListUsedAction in project druid by druid-io.

the class ActionBasedUsedSegmentChecker method findUsedSegments.

@Override
public Set<DataSegment> findUsedSegments(Set<SegmentIdentifier> identifiers) throws IOException {
    // Group by dataSource
    final Map<String, Set<SegmentIdentifier>> identifiersByDataSource = Maps.newTreeMap();
    for (SegmentIdentifier identifier : identifiers) {
        if (!identifiersByDataSource.containsKey(identifier.getDataSource())) {
            identifiersByDataSource.put(identifier.getDataSource(), Sets.<SegmentIdentifier>newHashSet());
        }
        identifiersByDataSource.get(identifier.getDataSource()).add(identifier);
    }
    final Set<DataSegment> retVal = Sets.newHashSet();
    for (Map.Entry<String, Set<SegmentIdentifier>> entry : identifiersByDataSource.entrySet()) {
        final List<Interval> intervals = JodaUtils.condenseIntervals(Iterables.transform(entry.getValue(), new Function<SegmentIdentifier, Interval>() {

            @Override
            public Interval apply(SegmentIdentifier input) {
                return input.getInterval();
            }
        }));
        final List<DataSegment> usedSegmentsForIntervals = taskActionClient.submit(new SegmentListUsedAction(entry.getKey(), null, intervals));
        for (DataSegment segment : usedSegmentsForIntervals) {
            if (identifiers.contains(SegmentIdentifier.fromDataSegment(segment))) {
                retVal.add(segment);
            }
        }
    }
    return retVal;
}

Example 7 with SegmentListUsedAction

use of io.druid.indexing.common.actions.SegmentListUsedAction in project druid by druid-io.

the class ActionBasedUsedSegmentCheckerTest method testBasic.

@Test
public void testBasic() throws IOException {
    final TaskActionClient taskActionClient = EasyMock.createMock(TaskActionClient.class);
    EasyMock.expect(taskActionClient.submit(new SegmentListUsedAction("bar", null, ImmutableList.of(new Interval("2002/P1D"))))).andReturn(ImmutableList.of(DataSegment.builder().dataSource("bar").interval(new Interval("2002/P1D")).shardSpec(new LinearShardSpec(0)).version("b").build(), DataSegment.builder().dataSource("bar").interval(new Interval("2002/P1D")).shardSpec(new LinearShardSpec(1)).version("b").build()));
    EasyMock.expect(taskActionClient.submit(new SegmentListUsedAction("foo", null, ImmutableList.of(new Interval("2000/P1D"), new Interval("2001/P1D"))))).andReturn(ImmutableList.of(DataSegment.builder().dataSource("foo").interval(new Interval("2000/P1D")).shardSpec(new LinearShardSpec(0)).version("a").build(), DataSegment.builder().dataSource("foo").interval(new Interval("2000/P1D")).shardSpec(new LinearShardSpec(1)).version("a").build(), DataSegment.builder().dataSource("foo").interval(new Interval("2001/P1D")).shardSpec(new LinearShardSpec(1)).version("b").build(), DataSegment.builder().dataSource("foo").interval(new Interval("2002/P1D")).shardSpec(new LinearShardSpec(1)).version("b").build()));
    EasyMock.replay(taskActionClient);
    final UsedSegmentChecker checker = new ActionBasedUsedSegmentChecker(taskActionClient);
    final Set<DataSegment> segments = checker.findUsedSegments(ImmutableSet.of(new SegmentIdentifier("foo", new Interval("2000/P1D"), "a", new LinearShardSpec(1)), new SegmentIdentifier("foo", new Interval("2001/P1D"), "b", new LinearShardSpec(0)), new SegmentIdentifier("bar", new Interval("2002/P1D"), "b", new LinearShardSpec(0))));
    Assert.assertEquals(ImmutableSet.of(DataSegment.builder().dataSource("foo").interval(new Interval("2000/P1D")).shardSpec(new LinearShardSpec(1)).version("a").build(), DataSegment.builder().dataSource("bar").interval(new Interval("2002/P1D")).shardSpec(new LinearShardSpec(0)).version("b").build()), segments);
    EasyMock.verify(taskActionClient);
}

Example 8 with SegmentListUsedAction

use of io.druid.indexing.common.actions.SegmentListUsedAction in project druid by druid-io.

the class SameIntervalMergeTask method run.

@Override
public TaskStatus run(TaskToolbox toolbox) throws Exception {
    final List<DataSegment> segments = toolbox.getTaskActionClient().submit(new SegmentListUsedAction(getDataSource(), getInterval(), null));
    SubTask mergeTask = new SubTask(getId(), getDataSource(), segments, aggregators, rollup, indexSpec, buildV9Directly, getContext());
    final TaskStatus status = mergeTask.run(toolbox);
    if (!status.isSuccess()) {
        return TaskStatus.fromCode(getId(), status.getStatusCode());
    }
    return success();
}

Example 9 with SegmentListUsedAction

use of io.druid.indexing.common.actions.SegmentListUsedAction in project druid by druid-io.

the class ConvertSegmentTask method convertSegment.

private static void convertSegment(TaskToolbox toolbox, final DataSegment segment, IndexSpec indexSpec, boolean force, boolean validate) throws SegmentLoadingException, IOException {
    log.info("Converting segment[%s]", segment);
    final TaskActionClient actionClient = toolbox.getTaskActionClient();
    final List<DataSegment> currentSegments = actionClient.submit(new SegmentListUsedAction(segment.getDataSource(), segment.getInterval(), null));
    for (DataSegment currentSegment : currentSegments) {
        final String version = currentSegment.getVersion();
        final Integer binaryVersion = currentSegment.getBinaryVersion();
        if (!force && (version.startsWith(segment.getVersion()) && CURR_VERSION_INTEGER.equals(binaryVersion))) {
            log.info("Skipping already updated segment[%s].", segment);
            return;
        }
    }
    final Map<DataSegment, File> localSegments = toolbox.fetchSegments(Collections.singletonList(segment));
    final File location = localSegments.get(segment);
    final File outLocation = new File(location, "v9_out");
    if (toolbox.getIndexIO().convertSegment(location, outLocation, indexSpec, force, validate)) {
        final int outVersion = IndexIO.getVersionFromDir(outLocation);
        // Appending to the version makes a new version that inherits most comparability parameters of the original
        // version, but is "newer" than said original version.
        DataSegment updatedSegment = segment.withVersion(String.format("%s_v%s", segment.getVersion(), outVersion));
        updatedSegment = toolbox.getSegmentPusher().push(outLocation, updatedSegment);
        actionClient.submit(new SegmentInsertAction(Sets.newHashSet(updatedSegment)));
    } else {
        log.info("Conversion failed.");
    }
}