Examples with HllSketch com.yahoo.sketches.hll.HllSketch used on opensource projects

Example 46 with HllSketch

use of com.yahoo.sketches.hll.HllSketch in project sketches-pig by DataSketches.

the class UnionSketchTest method algebraicFinalFromIntermediate.

@Test
public void algebraicFinalFromIntermediate() throws Exception {
    @SuppressWarnings("unchecked") EvalFunc<DataByteArray> func = (EvalFunc<DataByteArray>) Class.forName(new UnionSketch().getFinal()).newInstance();
    HllSketch inputSketch = new HllSketch(12);
    inputSketch.update("a");
    inputSketch.update("b");
    DataBag bag = bagFactory.newDefaultBag();
    bag.add(tupleFactory.newTuple(new DataByteArray(inputSketch.toCompactByteArray())));
    DataByteArray result = func.exec(tupleFactory.newTuple(bag));
    HllSketch sketch = DataToSketchTest.getSketch(result);
    Assert.assertFalse(sketch.isEmpty());
    Assert.assertEquals(sketch.getEstimate(), 2.0, 0.01);
}

Example 47 with HllSketch

use of com.yahoo.sketches.hll.HllSketch in project sketches-pig by DataSketches.

the class UnionSketchTest method execNullInputTuple.

@Test
public void execNullInputTuple() throws Exception {
    EvalFunc<DataByteArray> func = new UnionSketch();
    DataByteArray result = func.exec(null);
    HllSketch sketch = DataToSketchTest.getSketch(result);
    Assert.assertTrue(sketch.isEmpty());
}

Example 48 with HllSketch

use of com.yahoo.sketches.hll.HllSketch in project sketches-pig by DataSketches.

the class DataToSketch method exec.

/**
 * Top-level exec function.
 * This method accepts an input Tuple containing a Bag of one or more inner <b>Datum Tuples</b>
 * and returns a single serialized HllSketch as a DataByteArray.
 *
 * <b>Datum Tuple</b> is a Tuple containing a single field, which can be one of the following
 * (Java type: Pig type):
 * <ul>
 *   <li>Byte: BYTE</li>
 *   <li>Integer: INTEGER</li>
 *   <li>Long: LONG</li>
 *   <li>Float: FLOAT</li>
 *   <li>Double: DOUBLE</li>
 *   <li>String: CHARARRAY</li>
 *   <li>DataByteArray: BYTEARRAY</li>
 * </ul>
 *
 * @param inputTuple A tuple containing a single bag, containing Datum Tuples.
 * @return serialized HllSketch
 * @see "org.apache.pig.EvalFunc.exec(org.apache.pig.data.Tuple)"
 * @throws IOException from Pig.
 */
@Override
public DataByteArray exec(final Tuple inputTuple) throws IOException {
    if (isFirstCall_) {
        Logger.getLogger(getClass()).info("Exec was used");
        isFirstCall_ = false;
    }
    if (inputTuple == null || inputTuple.size() == 0) {
        if (emptySketch_ == null) {
            emptySketch_ = new DataByteArray(new HllSketch(lgK_, tgtHllType_).toCompactByteArray());
        }
        return emptySketch_;
    }
    final Union union = new Union(lgK_);
    final DataBag bag = (DataBag) inputTuple.get(0);
    updateUnion(bag, union);
    return new DataByteArray(union.getResult(tgtHllType_).toCompactByteArray());
}

Example 49 with HllSketch

use of com.yahoo.sketches.hll.HllSketch in project sketches-pig by DataSketches.

the class SketchToEstimate method exec.

@Override
public Double exec(final Tuple sketchTuple) throws IOException {
    if ((sketchTuple == null) || (sketchTuple.size() == 0)) {
        return null;
    }
    final DataByteArray dba = (DataByteArray) sketchTuple.get(0);
    final HllSketch sketch = HllSketch.wrap(Memory.wrap(dba.get()));
    return sketch.getEstimate();
}

Example 50 with HllSketch

use of com.yahoo.sketches.hll.HllSketch in project Gaffer by gchq.

the class HllSketchEntityGeneratorTest method shouldCreateSimpleEntities.

@Test
public void shouldCreateSimpleEntities() {
    // Given
    HllSketchEntityGenerator hllSketchEntityGenerator = new HllSketchEntityGenerator();
    Edge edge = new Edge.Builder().group(TestGroups.ENTITY).source(A).dest(B).build();
    List<? extends Element> edges = Arrays.asList(edge);
    // When
    Iterable<? extends Element> elements = hllSketchEntityGenerator.apply(edges);
    // Then
    Iterator<? extends Element> elementIterator = elements.iterator();
    Edge edgeResult = (Edge) elementIterator.next();
    Entity entityResultA = (Entity) elementIterator.next();
    Entity entityResultB = (Entity) elementIterator.next();
    assertThat(elementIterator.hasNext()).isFalse();
    assertThat(edgeResult).isEqualTo(edge);
    assertThat(entityResultA.getGroup()).isEqualTo(DEFAULT_ENTITY_GROUP);
    assertThat(entityResultA.getVertex()).isEqualTo(A);
    HllSketch entityCardinalityA = (HllSketch) entityResultA.getProperty(DEFAULT_PROPERTY_NAME);
    assertThat(entityCardinalityA.getEstimate()).isEqualTo(1);
    assertThat(entityResultB.getGroup()).isEqualTo(DEFAULT_ENTITY_GROUP);
    assertThat(entityResultB.getVertex()).isEqualTo(B);
    HllSketch entityCardinalityB = (HllSketch) entityResultB.getProperty(DEFAULT_PROPERTY_NAME);
    assertThat(entityCardinalityB.getEstimate()).isEqualTo(1);
}