Examples with HumanReadableBytes org.apache.druid.java.util.common.HumanReadableBytes used on opensource projects

Example 21 with HumanReadableBytes

use of org.apache.druid.java.util.common.HumanReadableBytes in project druid by druid-io.

the class MaxSizeSplitHintSpecTest method testSplitSmallInputsGroupingIntoLargerSplits.

@Test
public void testSplitSmallInputsGroupingIntoLargerSplits() {
    final int eachInputSize = 3;
    final MaxSizeSplitHintSpec splitHintSpec = new MaxSizeSplitHintSpec(new HumanReadableBytes(10L), 10_000);
    final Function<Integer, InputFileAttribute> inputAttributeExtractor = InputFileAttribute::new;
    final List<List<Integer>> splits = Lists.newArrayList(splitHintSpec.split(IntStream.generate(() -> eachInputSize).limit(10).iterator(), inputAttributeExtractor));
    Assert.assertEquals(4, splits.size());
    Assert.assertEquals(3, splits.get(0).size());
    Assert.assertEquals(3, splits.get(1).size());
    Assert.assertEquals(3, splits.get(2).size());
    Assert.assertEquals(1, splits.get(3).size());
}

Example 22 with HumanReadableBytes

use of org.apache.druid.java.util.common.HumanReadableBytes in project druid by druid-io.

the class MaxSizeSplitHintSpecTest method testSplitLargeInputsReturningSplitsOfSingleInput.

@Test
public void testSplitLargeInputsReturningSplitsOfSingleInput() {
    final int eachInputSize = 15;
    final MaxSizeSplitHintSpec splitHintSpec = new MaxSizeSplitHintSpec(new HumanReadableBytes(10L), 10_000);
    final Function<Integer, InputFileAttribute> inputAttributeExtractor = InputFileAttribute::new;
    final List<List<Integer>> splits = Lists.newArrayList(splitHintSpec.split(IntStream.generate(() -> eachInputSize).limit(10).iterator(), inputAttributeExtractor));
    Assert.assertEquals(10, splits.size());
    for (List<Integer> split : splits) {
        Assert.assertEquals(1, split.size());
    }
}

Example 23 with HumanReadableBytes

use of org.apache.druid.java.util.common.HumanReadableBytes in project druid by druid-io.

the class MaxSizeSplitHintSpecTest method testSplitSkippingEmptyInputs.

@Test
public void testSplitSkippingEmptyInputs() {
    final int nonEmptyInputSize = 3;
    final MaxSizeSplitHintSpec splitHintSpec = new MaxSizeSplitHintSpec(new HumanReadableBytes(10L), null);
    final Function<Integer, InputFileAttribute> inputAttributeExtractor = InputFileAttribute::new;
    final IntStream dataStream = IntStream.concat(IntStream.concat(IntStream.generate(() -> 0).limit(10), IntStream.generate(() -> nonEmptyInputSize).limit(10)), IntStream.generate(() -> 0).limit(10));
    final List<List<Integer>> splits = Lists.newArrayList(splitHintSpec.split(dataStream.iterator(), inputAttributeExtractor));
    Assert.assertEquals(4, splits.size());
    Assert.assertEquals(3, splits.get(0).size());
    Assert.assertEquals(3, splits.get(1).size());
    Assert.assertEquals(3, splits.get(2).size());
    Assert.assertEquals(1, splits.get(3).size());
}

Example 24 with HumanReadableBytes

use of org.apache.druid.java.util.common.HumanReadableBytes in project druid by druid-io.

the class CombiningInputSourceTest method testCreateSplits.

@Test
public void testCreateSplits() {
    final File file = EasyMock.niceMock(File.class);
    EasyMock.expect(file.length()).andReturn(30L).anyTimes();
    EasyMock.replay(file);
    final TestFileInputSource fileSource = new TestFileInputSource(generateFiles(3));
    final TestUriInputSource uriInputSource = new TestUriInputSource(ImmutableList.of(URI.create("http://test.com/http-test3"), URI.create("http://test.com/http-test4"), URI.create("http://test.com/http-test5")));
    final CombiningInputSource combiningInputSource = new CombiningInputSource(ImmutableList.of(fileSource, uriInputSource));
    List<InputSplit> combinedInputSplits = combiningInputSource.createSplits(new NoopInputFormat(), new MaxSizeSplitHintSpec(new HumanReadableBytes(5L), null)).collect(Collectors.toList());
    Assert.assertEquals(6, combinedInputSplits.size());
    for (int i = 0; i < 3; i++) {
        Pair<SplittableInputSource, InputSplit> splitPair = (Pair) combinedInputSplits.get(i).get();
        InputSplit<File> fileSplits = splitPair.rhs;
        Assert.assertTrue(splitPair.lhs instanceof TestFileInputSource);
        Assert.assertEquals(5, fileSplits.get().length());
    }
    for (int i = 3; i < combinedInputSplits.size(); i++) {
        Pair<SplittableInputSource, InputSplit> splitPair = (Pair) combinedInputSplits.get(i).get();
        InputSplit<URI> fileSplits = splitPair.rhs;
        Assert.assertTrue(splitPair.lhs instanceof TestUriInputSource);
        Assert.assertEquals(URI.create("http://test.com/http-test" + i), fileSplits.get());
    }
}

Example 25 with HumanReadableBytes

use of org.apache.druid.java.util.common.HumanReadableBytes in project druid by druid-io.

the class ClientCompactionTaskQuerySerdeTest method testClientCompactionTaskQueryToCompactionTask.

@Test
public void testClientCompactionTaskQueryToCompactionTask() throws IOException {
    final ObjectMapper mapper = setupInjectablesInObjectMapper(new DefaultObjectMapper());
    final ClientCompactionTaskQuery query = new ClientCompactionTaskQuery("id", "datasource", new ClientCompactionIOConfig(new ClientCompactionIntervalSpec(Intervals.of("2019/2020"), "testSha256OfSortedSegmentIds"), true), new ClientCompactionTaskQueryTuningConfig(null, 40000, 2000L, null, new SegmentsSplitHintSpec(new HumanReadableBytes(100000L), 10), new DynamicPartitionsSpec(100, 30000L), new IndexSpec(new DefaultBitmapSerdeFactory(), CompressionStrategy.LZ4, CompressionStrategy.LZF, LongEncodingStrategy.LONGS), new IndexSpec(new DefaultBitmapSerdeFactory(), CompressionStrategy.LZ4, CompressionStrategy.UNCOMPRESSED, LongEncodingStrategy.AUTO), 2, 1000L, TmpFileSegmentWriteOutMediumFactory.instance(), 100, 5, 1000L, new Duration(3000L), 7, 1000, 100), new ClientCompactionTaskGranularitySpec(Granularities.DAY, Granularities.HOUR, true), new ClientCompactionTaskDimensionsSpec(DimensionsSpec.getDefaultSchemas(ImmutableList.of("ts", "dim"))), new AggregatorFactory[] { new CountAggregatorFactory("cnt") }, new ClientCompactionTaskTransformSpec(new SelectorDimFilter("dim1", "foo", null)), ImmutableMap.of("key", "value"));
    final byte[] json = mapper.writeValueAsBytes(query);
    final CompactionTask task = (CompactionTask) mapper.readValue(json, Task.class);
    Assert.assertEquals(query.getId(), task.getId());
    Assert.assertEquals(query.getDataSource(), task.getDataSource());
    Assert.assertTrue(task.getIoConfig().getInputSpec() instanceof CompactionIntervalSpec);
    Assert.assertEquals(query.getIoConfig().getInputSpec().getInterval(), ((CompactionIntervalSpec) task.getIoConfig().getInputSpec()).getInterval());
    Assert.assertEquals(query.getIoConfig().getInputSpec().getSha256OfSortedSegmentIds(), ((CompactionIntervalSpec) task.getIoConfig().getInputSpec()).getSha256OfSortedSegmentIds());
    Assert.assertEquals(query.getTuningConfig().getMaxRowsInMemory().intValue(), task.getTuningConfig().getMaxRowsInMemory());
    Assert.assertEquals(query.getTuningConfig().getMaxBytesInMemory().longValue(), task.getTuningConfig().getMaxBytesInMemory());
    Assert.assertEquals(query.getTuningConfig().getSplitHintSpec(), task.getTuningConfig().getSplitHintSpec());
    Assert.assertEquals(query.getTuningConfig().getPartitionsSpec(), task.getTuningConfig().getPartitionsSpec());
    Assert.assertEquals(query.getTuningConfig().getIndexSpec(), task.getTuningConfig().getIndexSpec());
    Assert.assertEquals(query.getTuningConfig().getIndexSpecForIntermediatePersists(), task.getTuningConfig().getIndexSpecForIntermediatePersists());
    Assert.assertEquals(query.getTuningConfig().getPushTimeout().longValue(), task.getTuningConfig().getPushTimeout());
    Assert.assertEquals(query.getTuningConfig().getSegmentWriteOutMediumFactory(), task.getTuningConfig().getSegmentWriteOutMediumFactory());
    Assert.assertEquals(query.getTuningConfig().getMaxNumConcurrentSubTasks().intValue(), task.getTuningConfig().getMaxNumConcurrentSubTasks());
    Assert.assertEquals(query.getTuningConfig().getMaxRetry().intValue(), task.getTuningConfig().getMaxRetry());
    Assert.assertEquals(query.getTuningConfig().getTaskStatusCheckPeriodMs().longValue(), task.getTuningConfig().getTaskStatusCheckPeriodMs());
    Assert.assertEquals(query.getTuningConfig().getChatHandlerTimeout(), task.getTuningConfig().getChatHandlerTimeout());
    Assert.assertEquals(query.getTuningConfig().getMaxNumSegmentsToMerge().intValue(), task.getTuningConfig().getMaxNumSegmentsToMerge());
    Assert.assertEquals(query.getTuningConfig().getTotalNumMergeTasks().intValue(), task.getTuningConfig().getTotalNumMergeTasks());
    Assert.assertEquals(query.getGranularitySpec(), task.getGranularitySpec());
    Assert.assertEquals(query.getGranularitySpec().getQueryGranularity(), task.getGranularitySpec().getQueryGranularity());
    Assert.assertEquals(query.getGranularitySpec().getSegmentGranularity(), task.getGranularitySpec().getSegmentGranularity());
    Assert.assertEquals(query.getGranularitySpec().isRollup(), task.getGranularitySpec().isRollup());
    Assert.assertEquals(query.getIoConfig().isDropExisting(), task.getIoConfig().isDropExisting());
    Assert.assertEquals(query.getContext(), task.getContext());
    Assert.assertEquals(query.getDimensionsSpec().getDimensions(), task.getDimensionsSpec().getDimensions());
    Assert.assertEquals(query.getTransformSpec().getFilter(), task.getTransformSpec().getFilter());
    Assert.assertArrayEquals(query.getMetricsSpec(), task.getMetricsSpec());
}