Examples with ShardSpec org.apache.druid.timeline.partition.ShardSpec used on opensource projects

Example 16 with ShardSpec

use of org.apache.druid.timeline.partition.ShardSpec in project druid by druid-io.

the class SegmentAllocateActionTest method testWithPartialShardSpecAndOvershadowingSegments.

@Test
public void testWithPartialShardSpecAndOvershadowingSegments() throws IOException {
    final Task task = NoopTask.create();
    taskActionTestKit.getTaskLockbox().add(task);
    final ObjectMapper objectMapper = new DefaultObjectMapper();
    taskActionTestKit.getMetadataStorageCoordinator().announceHistoricalSegments(ImmutableSet.of(DataSegment.builder().dataSource(DATA_SOURCE).interval(Granularities.HOUR.bucket(PARTY_TIME)).version(PARTY_TIME.toString()).shardSpec(new HashBasedNumberedShardSpec(0, 2, 0, 2, ImmutableList.of("dim1"), null, objectMapper)).size(0).build(), DataSegment.builder().dataSource(DATA_SOURCE).interval(Granularities.HOUR.bucket(PARTY_TIME)).version(PARTY_TIME.toString()).shardSpec(new HashBasedNumberedShardSpec(1, 2, 1, 2, ImmutableList.of("dim1"), null, objectMapper)).size(0).build()));
    final SegmentAllocateAction action = new SegmentAllocateAction(DATA_SOURCE, PARTY_TIME, Granularities.MINUTE, Granularities.HOUR, "seq", null, true, new HashBasedNumberedPartialShardSpec(ImmutableList.of("dim1"), 1, 2, null), lockGranularity, null);
    final SegmentIdWithShardSpec segmentIdentifier = action.perform(task, taskActionTestKit.getTaskActionToolbox());
    Assert.assertNotNull(segmentIdentifier);
    final ShardSpec shardSpec = segmentIdentifier.getShardSpec();
    Assert.assertEquals(2, shardSpec.getPartitionNum());
    Assert.assertTrue(shardSpec instanceof HashBasedNumberedShardSpec);
    final HashBasedNumberedShardSpec hashBasedNumberedShardSpec = (HashBasedNumberedShardSpec) shardSpec;
    Assert.assertEquals(2, hashBasedNumberedShardSpec.getNumCorePartitions());
    Assert.assertEquals(ImmutableList.of("dim1"), hashBasedNumberedShardSpec.getPartitionDimensions());
}

Example 17 with ShardSpec

use of org.apache.druid.timeline.partition.ShardSpec in project druid by druid-io.

the class DimFilterUtilsTest method testFilterShards.

@Test
public void testFilterShards() {
    DimFilter filter1 = EasyMock.createMock(DimFilter.class);
    EasyMock.expect(filter1.getDimensionRangeSet("dim1")).andReturn(rangeSet(ImmutableList.of(Range.lessThan("abc")))).anyTimes();
    EasyMock.expect(filter1.getDimensionRangeSet("dim2")).andReturn(null).anyTimes();
    ShardSpec shard1 = shardSpec("dim1", true);
    ShardSpec shard2 = shardSpec("dim1", false);
    ShardSpec shard3 = shardSpec("dim1", false);
    ShardSpec shard4 = shardSpec("dim2", false);
    ShardSpec shard5 = shardSpec("dim2", false);
    ShardSpec shard6 = shardSpec("dim2", false);
    ShardSpec shard7 = shardSpec("dim2", false);
    List<ShardSpec> shards = ImmutableList.of(shard1, shard2, shard3, shard4, shard5, shard6, shard7);
    EasyMock.replay(filter1, shard1, shard2, shard3, shard4, shard5, shard6, shard7);
    Set<ShardSpec> expected1 = ImmutableSet.of(shard1, shard4, shard5, shard6, shard7);
    assertFilterResult(filter1, shards, expected1);
}

Example 18 with ShardSpec

use of org.apache.druid.timeline.partition.ShardSpec in project druid by druid-io.

the class DimFilterUtilsTest method shardSpec.

private static ShardSpec shardSpec(String dimension, boolean contained) {
    ShardSpec shard = EasyMock.createMock(ShardSpec.class);
    EasyMock.expect(shard.getDomainDimensions()).andReturn(ImmutableList.of(dimension)).anyTimes();
    EasyMock.expect(shard.possibleInDomain(EasyMock.anyObject())).andReturn(contained).anyTimes();
    return shard;
}

Example 19 with ShardSpec

use of org.apache.druid.timeline.partition.ShardSpec in project druid by druid-io.

the class DimFilterUtilsTest method assertFilterResult.

private void assertFilterResult(DimFilter filter, Iterable<ShardSpec> input, Set<ShardSpec> expected) {
    Set<ShardSpec> result = DimFilterUtils.filterShards(filter, input, CONVERTER);
    Assert.assertEquals(expected, result);
    Map<String, Optional<RangeSet<String>>> dimensionRangeMap = new HashMap<>();
    result = new HashSet<>();
    for (ShardSpec shard : input) {
        result.addAll(DimFilterUtils.filterShards(filter, ImmutableList.of(shard), CONVERTER, dimensionRangeMap));
    }
    Assert.assertEquals(expected, result);
}

Example 20 with ShardSpec

use of org.apache.druid.timeline.partition.ShardSpec in project druid by druid-io.

the class DimFilterUtils method filterShards.

/**
 * Filter the given iterable of objects by removing any object whose ShardSpec, obtained from the converter function,
 * does not fit in the RangeSet of the dimFilter {@link DimFilter#getDimensionRangeSet(String)}. The returned set
 * contains the filtered objects in the same order as they appear in input.
 *
 * DimensionRangedCache stores the RangeSets of different dimensions for the dimFilter. It should be re-used
 * between calls with the same dimFilter to save redundant calls of {@link DimFilter#getDimensionRangeSet(String)}
 * on same dimensions.
 *
 * @param dimFilter           The filter to use
 * @param input               The iterable of objects to be filtered
 * @param converter           The function to convert T to ShardSpec that can be filtered by
 * @param dimensionRangeCache The cache of RangeSets of different dimensions for the dimFilter
 * @param <T>                 This can be any type, as long as transform function is provided to convert this to ShardSpec
 *
 * @return The set of filtered object, in the same order as input
 */
public static <T> Set<T> filterShards(final DimFilter dimFilter, final Iterable<T> input, final Function<T, ShardSpec> converter, final Map<String, Optional<RangeSet<String>>> dimensionRangeCache) {
    Set<T> retSet = new LinkedHashSet<>();
    for (T obj : input) {
        ShardSpec shard = converter.apply(obj);
        boolean include = true;
        if (dimFilter != null && shard != null) {
            Map<String, RangeSet<String>> filterDomain = new HashMap<>();
            List<String> dimensions = shard.getDomainDimensions();
            for (String dimension : dimensions) {
                Optional<RangeSet<String>> optFilterRangeSet = dimensionRangeCache.computeIfAbsent(dimension, d -> Optional.ofNullable(dimFilter.getDimensionRangeSet(d)));
                if (optFilterRangeSet.isPresent()) {
                    filterDomain.put(dimension, optFilterRangeSet.get());
                }
            }
            if (!filterDomain.isEmpty() && !shard.possibleInDomain(filterDomain)) {
                include = false;
            }
        }
        if (include) {
            retSet.add(obj);
        }
    }
    return retSet;
}