Examples with ArrayOfDoublesUpdatableSketch org.apache.datasketches.tuple.arrayofdoubles.ArrayOfDoublesUpdatableSketch used on opensource projects

Example 11 with ArrayOfDoublesUpdatableSketch

use of org.apache.datasketches.tuple.arrayofdoubles.ArrayOfDoublesUpdatableSketch in project druid by druid-io.

the class ArrayOfDoublesSketchToQuantilesSketchPostAggregatorTest method testComparator.

@Test
public void testComparator() {
    ArrayOfDoublesUpdatableSketch s1 = new ArrayOfDoublesUpdatableSketchBuilder().setNominalEntries(16).setNumberOfValues(2).build();
    s1.update("foo", new double[] { 1.0, 2.0 });
    ArrayOfDoublesUpdatableSketch s2 = new ArrayOfDoublesUpdatableSketchBuilder().setNominalEntries(16).setNumberOfValues(2).build();
    s2.update("foo", new double[] { 2.0, 2.0 });
    s2.update("bar", new double[] { 3.0, 4.0 });
    PostAggregator field1 = EasyMock.createMock(PostAggregator.class);
    EasyMock.expect(field1.compute(EasyMock.anyObject(Map.class))).andReturn(s1).anyTimes();
    PostAggregator field2 = EasyMock.createMock(PostAggregator.class);
    EasyMock.expect(field2.compute(EasyMock.anyObject(Map.class))).andReturn(s2).anyTimes();
    EasyMock.replay(field1, field2);
    final ArrayOfDoublesSketchToQuantilesSketchPostAggregator postAgg1 = new ArrayOfDoublesSketchToQuantilesSketchPostAggregator("a", field1, null, null);
    final ArrayOfDoublesSketchToQuantilesSketchPostAggregator postAgg2 = new ArrayOfDoublesSketchToQuantilesSketchPostAggregator("a", field2, null, null);
    Comparator comparator = postAgg1.getComparator();
    DoublesSketch sketch1 = postAgg1.compute(ImmutableMap.of());
    DoublesSketch sketch2 = postAgg2.compute(ImmutableMap.of());
    // comparator compares value of getN, which is 1 for sketch1 and 2 for sketch2
    Assert.assertEquals(-1, comparator.compare(sketch1, sketch2));
}

Example 12 with ArrayOfDoublesUpdatableSketch

use of org.apache.datasketches.tuple.arrayofdoubles.ArrayOfDoublesUpdatableSketch in project druid by druid-io.

the class GenerateTestData method generateSketches.

private static void generateSketches() throws Exception {
    Path path = FileSystems.getDefault().getPath("array_of_doubles_sketch_data.tsv");
    try (BufferedWriter out = Files.newBufferedWriter(path, StandardCharsets.UTF_8)) {
        Random rand = ThreadLocalRandom.current();
        int key = 0;
        for (int i = 0; i < 20; i++) {
            ArrayOfDoublesUpdatableSketch sketch = new ArrayOfDoublesUpdatableSketchBuilder().setNominalEntries(1024).build();
            sketch.update(key++, new double[] { 1 });
            sketch.update(key++, new double[] { 1 });
            out.write("2015010101");
            out.write('\t');
            out.write("product_" + (rand.nextInt(10) + 1));
            out.write('\t');
            out.write(StringUtils.encodeBase64String(sketch.compact().toByteArray()));
            out.newLine();
        }
    }
}

Example 13 with ArrayOfDoublesUpdatableSketch

use of org.apache.datasketches.tuple.arrayofdoubles.ArrayOfDoublesUpdatableSketch in project druid by druid-io.

the class ArrayOfDoublesSketchBuildBufferAggregator method aggregate.

@Override
public void aggregate(final ByteBuffer buf, final int position) {
    for (int i = 0; i < valueSelectors.length; i++) {
        if (valueSelectors[i].isNull()) {
            return;
        } else {
            values[i] = valueSelectors[i].getDouble();
        }
    }
    // Wrapping memory and ArrayOfDoublesSketch is inexpensive compared to sketch operations.
    // Maintaining a cache of wrapped objects per buffer position like in Theta sketch aggregator
    // might might be considered, but it would increase complexity including relocate() support.
    final WritableMemory mem = WritableMemory.writableWrap(buf, ByteOrder.LITTLE_ENDIAN);
    final WritableMemory region = mem.writableRegion(position, maxIntermediateSize);
    final ArrayOfDoublesUpdatableSketch sketch = ArrayOfDoublesSketches.wrapUpdatableSketch(region);
    final IndexedInts keys = keySelector.getRow();
    if (canLookupUtf8) {
        for (int i = 0, keysSize = keys.size(); i < keysSize; i++) {
            final ByteBuffer key;
            if (canCacheById) {
                key = (ByteBuffer) stringCache.computeIfAbsent(keys.get(i), keySelector::lookupNameUtf8);
            } else {
                key = keySelector.lookupNameUtf8(keys.get(i));
            }
            if (key != null) {
                byte[] bytes = new byte[key.remaining()];
                key.mark();
                key.get(bytes);
                key.reset();
                sketch.update(bytes, values);
            }
        }
    } else {
        for (int i = 0, keysSize = keys.size(); i < keysSize; i++) {
            final String key;
            if (canCacheById) {
                key = (String) stringCache.computeIfAbsent(keys.get(i), keySelector::lookupName);
            } else {
                key = keySelector.lookupName(keys.get(i));
            }
            sketch.update(key, values);
        }
    }
}