Examples with ShardSpec io.druid.timeline.partition.ShardSpec used on opensource projects

Example 16 with ShardSpec

use of io.druid.timeline.partition.ShardSpec in project druid by druid-io.

the class DimFilterUtilsTest method testFilterShards.

@Test
public void testFilterShards() {
    DimFilter filter1 = EasyMock.createMock(DimFilter.class);
    EasyMock.expect(filter1.getDimensionRangeSet("dim1")).andReturn(rangeSet(ImmutableList.of(Range.lessThan("abc")))).anyTimes();
    EasyMock.expect(filter1.getDimensionRangeSet("dim2")).andReturn(null).anyTimes();
    DimFilter filter2 = EasyMock.createMock(DimFilter.class);
    EasyMock.expect(filter2.getDimensionRangeSet("dim1")).andReturn(rangeSet(ImmutableList.of(Range.singleton("e")))).anyTimes();
    EasyMock.expect(filter2.getDimensionRangeSet("dim2")).andReturn(rangeSet(ImmutableList.of(Range.singleton("na")))).anyTimes();
    ShardSpec shard1 = shardSpec("dim1", Range.atMost("abc"));
    ShardSpec shard2 = shardSpec("dim1", Range.closed("abc", "def"));
    ShardSpec shard3 = shardSpec("dim1", Range.atLeast("def"));
    ShardSpec shard4 = shardSpec("dim2", Range.atMost("hello"));
    ShardSpec shard5 = shardSpec("dim2", Range.closed("hello", "jk"));
    ShardSpec shard6 = shardSpec("dim2", Range.closed("jk", "na"));
    ShardSpec shard7 = shardSpec("dim2", Range.atLeast("na"));
    List<ShardSpec> shards = ImmutableList.of(shard1, shard2, shard3, shard4, shard5, shard6, shard7);
    EasyMock.replay(filter1, filter2, shard1, shard2, shard3, shard4, shard5, shard6, shard7);
    Set<ShardSpec> expected1 = ImmutableSet.of(shard1, shard4, shard5, shard6, shard7);
    assertFilterResult(filter1, shards, expected1);
    Set<ShardSpec> expected2 = ImmutableSet.of(shard3, shard6, shard7);
    assertFilterResult(filter2, shards, expected2);
}

Example 17 with ShardSpec

use of io.druid.timeline.partition.ShardSpec in project druid by druid-io.

the class DimFilterUtilsTest method assertFilterResult.

private void assertFilterResult(DimFilter filter, Iterable<ShardSpec> input, Set<ShardSpec> expected) {
    Set<ShardSpec> result = DimFilterUtils.filterShards(filter, input, CONVERTER);
    Assert.assertEquals(expected, result);
    Map<String, Optional<RangeSet<String>>> dimensionRangeMap = Maps.newHashMap();
    result = Sets.newHashSet();
    for (ShardSpec shard : input) {
        result.addAll(DimFilterUtils.filterShards(filter, ImmutableList.of(shard), CONVERTER, dimensionRangeMap));
    }
    Assert.assertEquals(expected, result);
}

Example 18 with ShardSpec

use of io.druid.timeline.partition.ShardSpec in project druid by druid-io.

the class DimFilterUtils method filterShards.

/**
   * Filter the given iterable of objects by removing any object whose ShardSpec, obtained from the converter function,
   * does not fit in the RangeSet of the dimFilter {@link DimFilter#getDimensionRangeSet(String)}. The returned set
   * contains the filtered objects in the same order as they appear in input.
   *
   * DimensionRangedCache stores the RangeSets of different dimensions for the dimFilter. It should be re-used
   * between calls with the same dimFilter to save redundant calls of {@link DimFilter#getDimensionRangeSet(String)}
   * on same dimensions.
   *
   * @param dimFilter The filter to use
   * @param input The iterable of objects to be filtered
   * @param converter The function to convert T to ShardSpec that can be filtered by
   * @param dimensionRangeCache The cache of RangeSets of different dimensions for the dimFilter
   * @param <T> This can be any type, as long as transform function is provided to convert this to ShardSpec
   * @return The set of filtered object, in the same order as input
   */
public static <T> Set<T> filterShards(DimFilter dimFilter, Iterable<T> input, Function<T, ShardSpec> converter, Map<String, Optional<RangeSet<String>>> dimensionRangeCache) {
    Set<T> retSet = Sets.newLinkedHashSet();
    for (T obj : input) {
        ShardSpec shard = converter.apply(obj);
        boolean include = true;
        if (dimFilter != null && shard != null) {
            Map<String, Range<String>> domain = shard.getDomain();
            for (Map.Entry<String, Range<String>> entry : domain.entrySet()) {
                Optional<RangeSet<String>> optFilterRangeSet = dimensionRangeCache.get(entry.getKey());
                if (optFilterRangeSet == null) {
                    RangeSet<String> filterRangeSet = dimFilter.getDimensionRangeSet(entry.getKey());
                    optFilterRangeSet = Optional.fromNullable(filterRangeSet);
                    dimensionRangeCache.put(entry.getKey(), optFilterRangeSet);
                }
                if (optFilterRangeSet.isPresent() && optFilterRangeSet.get().subRangeSet(entry.getValue()).isEmpty()) {
                    include = false;
                }
            }
        }
        if (include) {
            retSet.add(obj);
        }
    }
    return retSet;
}