Examples with MapBasedRow org.apache.druid.data.input.MapBasedRow used on opensource projects

Example 6 with MapBasedRow

use of org.apache.druid.data.input.MapBasedRow in project druid by druid-io.

the class LimitedBufferHashGrouperTest method testMinBufferSize.

@Test
public void testMinBufferSize() {
    final TestColumnSelectorFactory columnSelectorFactory = GrouperTestUtil.newColumnSelectorFactory();
    final LimitedBufferHashGrouper<Integer> grouper = makeGrouper(columnSelectorFactory, 12120);
    columnSelectorFactory.setRow(new MapBasedRow(0, ImmutableMap.of("value", 10L)));
    for (int i = 0; i < NUM_ROWS; i++) {
        Assert.assertTrue(String.valueOf(i + KEY_BASE), grouper.aggregate(i + KEY_BASE).isOk());
    }
    // With minimum buffer size, after the first swap, every new key added will result in a swap
    if (NullHandling.replaceWithDefault()) {
        Assert.assertEquals(224, grouper.getGrowthCount());
        Assert.assertEquals(104, grouper.getSize());
        Assert.assertEquals(209, grouper.getBuckets());
        Assert.assertEquals(104, grouper.getMaxSize());
    } else {
        Assert.assertEquals(899, grouper.getGrowthCount());
        Assert.assertEquals(101, grouper.getSize());
        Assert.assertEquals(202, grouper.getBuckets());
        Assert.assertEquals(101, grouper.getMaxSize());
    }
    Assert.assertEquals(100, grouper.getLimit());
    // Aggregate slightly different row
    // Since these keys are smaller, they will evict the previous 100 top entries
    // First 100 of these new rows will be the expected results.
    columnSelectorFactory.setRow(new MapBasedRow(0, ImmutableMap.of("value", 11L)));
    for (int i = 0; i < NUM_ROWS; i++) {
        Assert.assertTrue(String.valueOf(i), grouper.aggregate(i).isOk());
    }
    if (NullHandling.replaceWithDefault()) {
        Assert.assertEquals(474, grouper.getGrowthCount());
        Assert.assertEquals(104, grouper.getSize());
        Assert.assertEquals(209, grouper.getBuckets());
        Assert.assertEquals(104, grouper.getMaxSize());
    } else {
        Assert.assertEquals(1899, grouper.getGrowthCount());
        Assert.assertEquals(101, grouper.getSize());
        Assert.assertEquals(202, grouper.getBuckets());
        Assert.assertEquals(101, grouper.getMaxSize());
    }
    Assert.assertEquals(100, grouper.getLimit());
    final List<Grouper.Entry<Integer>> expected = new ArrayList<>();
    for (int i = 0; i < LIMIT; i++) {
        expected.add(new Grouper.Entry<>(i, new Object[] { 11L, 1L }));
    }
    Assert.assertEquals(expected, Lists.newArrayList(grouper.iterator(true)));
    // iterate again, even though the min-max offset heap has been destroyed, it is replaced with a reverse sorted array
    Assert.assertEquals(expected, Lists.newArrayList(grouper.iterator(true)));
}

Example 7 with MapBasedRow

use of org.apache.druid.data.input.MapBasedRow in project druid by druid-io.

the class LimitedBufferHashGrouperTest method testIteratorOrderByDimDesc.

@Test
public void testIteratorOrderByDimDesc() {
    final TestColumnSelectorFactory columnSelectorFactory = GrouperTestUtil.newColumnSelectorFactory();
    final LimitedBufferHashGrouper<Integer> grouper = makeGrouperWithOrderBy(columnSelectorFactory, "value", OrderByColumnSpec.Direction.DESCENDING);
    for (int i = 0; i < NUM_ROWS; i++) {
        // limited grouper iterator will always sort by keys in ascending order, even if the heap was sorted by values
        // so, we aggregate with keys and values both ascending so that the results are not re-ordered by key
        columnSelectorFactory.setRow(new MapBasedRow(0, ImmutableMap.of("value", i + 1)));
        Assert.assertTrue(String.valueOf(i + KEY_BASE), grouper.aggregate(i + KEY_BASE).isOk());
    }
    List<Grouper.Entry<Integer>> iterated = Lists.newArrayList(grouper.iterator(true));
    Assert.assertEquals(LIMIT, iterated.size());
    for (int i = 0; i < LIMIT; i++) {
        Assert.assertEquals((long) NUM_ROWS - i, iterated.get(i).getValues()[0]);
    }
}

Example 8 with MapBasedRow

use of org.apache.druid.data.input.MapBasedRow in project druid by druid-io.

the class StreamingMergeSortedGrouperTest method testTimeout.

@Test
public void testTimeout() {
    expectedException.expect(QueryTimeoutException.class);
    final TestColumnSelectorFactory columnSelectorFactory = GrouperTestUtil.newColumnSelectorFactory();
    final StreamingMergeSortedGrouper<Integer> grouper = newGrouper(columnSelectorFactory, 100);
    columnSelectorFactory.setRow(new MapBasedRow(0, ImmutableMap.of("value", 10L)));
    grouper.aggregate(6);
    grouper.iterator();
}

Example 9 with MapBasedRow

use of org.apache.druid.data.input.MapBasedRow in project druid by druid-io.

the class StreamingMergeSortedGrouperTest method testAggregate.

@Test
public void testAggregate() {
    final TestColumnSelectorFactory columnSelectorFactory = GrouperTestUtil.newColumnSelectorFactory();
    final StreamingMergeSortedGrouper<Integer> grouper = newGrouper(columnSelectorFactory, 1024);
    columnSelectorFactory.setRow(new MapBasedRow(0, ImmutableMap.of("value", 10L)));
    grouper.aggregate(6);
    grouper.aggregate(6);
    grouper.aggregate(6);
    grouper.aggregate(10);
    grouper.aggregate(12);
    grouper.aggregate(12);
    grouper.finish();
    final List<Entry<Integer>> expected = ImmutableList.of(new Grouper.Entry<>(6, new Object[] { 30L, 3L }), new Grouper.Entry<>(10, new Object[] { 10L, 1L }), new Grouper.Entry<>(12, new Object[] { 20L, 2L }));
    final List<Entry<Integer>> unsortedEntries = Lists.newArrayList(grouper.iterator(true));
    Assert.assertEquals(expected, unsortedEntries);
}

Example 10 with MapBasedRow

use of org.apache.druid.data.input.MapBasedRow in project druid by druid-io.

the class StreamingMergeSortedGrouperTest method testStreamingAggregate.

private void testStreamingAggregate(int bufferSize) throws ExecutionException, InterruptedException {
    final ExecutorService exec = Execs.multiThreaded(2, "merge-sorted-grouper-test-%d");
    final TestColumnSelectorFactory columnSelectorFactory = GrouperTestUtil.newColumnSelectorFactory();
    final StreamingMergeSortedGrouper<Integer> grouper = newGrouper(columnSelectorFactory, bufferSize);
    final List<Entry<Integer>> expected = new ArrayList<>(1024);
    for (int i = 0; i < 1024; i++) {
        expected.add(new Entry<>(i, new Object[] { 100L, 10L }));
    }
    try {
        final Future future = exec.submit(() -> {
            columnSelectorFactory.setRow(new MapBasedRow(0, ImmutableMap.of("value", 10L)));
            for (int i = 0; i < 1024; i++) {
                for (int j = 0; j < 10; j++) {
                    grouper.aggregate(i);
                }
            }
            grouper.finish();
        });
        final List<Entry<Integer>> unsortedEntries = Lists.newArrayList(grouper.iterator(true));
        final List<Entry<Integer>> actual = Ordering.from((Comparator<Entry<Integer>>) (o1, o2) -> Ints.compare(o1.getKey(), o2.getKey())).sortedCopy(unsortedEntries);
        if (!actual.equals(expected)) {
            // Check there is an exception occured
            future.get();
            Assert.fail();
        }
    } finally {
        exec.shutdownNow();
    }
}