Examples with ItemsSketch com.yahoo.sketches.frequencies.ItemsSketch used on opensource projects

Example 6 with ItemsSketch

use of com.yahoo.sketches.frequencies.ItemsSketch in project sketches-pig by DataSketches.

the class DataToFrequentStringsSketchTest method algebraicIntermediateFinal.

@Test
public void algebraicIntermediateFinal() throws Exception {
    EvalFunc<Tuple> func = new DataToFrequentStringsSketch.IntermediateFinal("8");
    Tuple inputTuple = TupleFactory.getInstance().newTuple(1);
    DataBag bag = BagFactory.getInstance().newDefaultBag();
    inputTuple.set(0, bag);
    // this is to simulate the output from Initial
    bag.add(PigUtil.objectsToTuple(PigUtil.tuplesToBag(PigUtil.objectsToTuple("a"))));
    // this is to simulate the output from a prior call of IntermediateFinal
    ItemsSketch<String> s = new ItemsSketch<String>(8);
    s.update("b", 1L);
    s.update("a", 2L);
    s.update("b", 3L);
    bag.add(PigUtil.objectsToTuple(new DataByteArray(s.toByteArray(new ArrayOfStringsSerDe()))));
    Tuple resultTuple = func.exec(inputTuple);
    Assert.assertNotNull(resultTuple);
    Assert.assertEquals(resultTuple.size(), 1);
    DataByteArray bytes = (DataByteArray) resultTuple.get(0);
    Assert.assertTrue(bytes.size() > 0);
    ItemsSketch<String> sketch = ItemsSketch.getInstance(Memory.wrap(bytes.get()), new ArrayOfStringsSerDe());
    Assert.assertEquals(sketch.getNumActiveItems(), 2);
    Assert.assertEquals(sketch.getEstimate("a"), 3);
    Assert.assertEquals(sketch.getEstimate("b"), 4);
}

Example 7 with ItemsSketch

use of com.yahoo.sketches.frequencies.ItemsSketch in project sketches-pig by DataSketches.

the class FrequentStringsSketchToEstimatesTest method emptySketch.

@Test
public void emptySketch() throws Exception {
    EvalFunc<DataBag> func = new FrequentStringsSketchToEstimates();
    ItemsSketch<String> sketch = new ItemsSketch<String>(8);
    Tuple inputTuple = PigUtil.objectsToTuple(new DataByteArray(sketch.toByteArray(new ArrayOfStringsSerDe())));
    DataBag bag = func.exec(inputTuple);
    Assert.assertNotNull(bag);
    Assert.assertEquals(bag.size(), 0);
}

Example 8 with ItemsSketch

use of com.yahoo.sketches.frequencies.ItemsSketch in project sketches-pig by DataSketches.

the class UnionFrequentStringsSketchTest method algebraicIntemediateFinalEstimation.

@Test
public void algebraicIntemediateFinalEstimation() throws Exception {
    EvalFunc<Tuple> func = new UnionFrequentStringsSketch.IntermediateFinal("8");
    DataBag bag = BagFactory.getInstance().newDefaultBag();
    // this is to simulate the output from Initial
    {
        ItemsSketch<String> sketch = new ItemsSketch<String>(8);
        sketch.update("a", 10);
        sketch.update("b");
        sketch.update("c");
        sketch.update("d");
        sketch.update("e");
        sketch.update("f");
        sketch.update("g");
        sketch.update("g");
        DataBag innerBag = PigUtil.tuplesToBag(PigUtil.objectsToTuple(new DataByteArray(sketch.toByteArray(new ArrayOfStringsSerDe()))));
        bag.add(PigUtil.objectsToTuple(innerBag));
    }
    // this is to simulate the output from a prior call of IntermediateFinal
    {
        ItemsSketch<String> sketch = new ItemsSketch<String>(8);
        sketch.update("a");
        sketch.update("a");
        sketch.update("g", 5);
        sketch.update("h");
        sketch.update("i");
        sketch.update("j");
        sketch.update("k");
        sketch.update("l");
        bag.add(PigUtil.objectsToTuple(new DataByteArray(sketch.toByteArray(new ArrayOfStringsSerDe()))));
    }
    Tuple resultTuple = func.exec(PigUtil.objectsToTuple(bag));
    Assert.assertNotNull(resultTuple);
    Assert.assertEquals(resultTuple.size(), 1);
    DataByteArray bytes = (DataByteArray) resultTuple.get(0);
    Assert.assertTrue(bytes.size() > 0);
    ItemsSketch<String> sketch = ItemsSketch.getInstance(Memory.wrap(bytes.get()), new ArrayOfStringsSerDe());
    Assert.assertFalse(sketch.isEmpty());
    Assert.assertEquals(sketch.getStreamLength(), 29);
    ItemsSketch.Row<String>[] items = sketch.getFrequentItems(ErrorType.NO_FALSE_POSITIVES);
    Assert.assertEquals(items.length, 2);
    // only 2 items ("a" and "g") should have counts more than 1
    int count = 0;
    for (ItemsSketch.Row<String> item : items) {
        if (item.getLowerBound() > 1)
            count++;
    }
    Assert.assertEquals(count, 2);
}

Example 9 with ItemsSketch

use of com.yahoo.sketches.frequencies.ItemsSketch in project sketches-pig by DataSketches.

the class UnionFrequentStringsSketchTest method algebraicIntemediateFinalExact.

@Test
public void algebraicIntemediateFinalExact() throws Exception {
    EvalFunc<Tuple> func = new UnionFrequentStringsSketch.IntermediateFinal("8");
    DataBag bag = BagFactory.getInstance().newDefaultBag();
    // this is to simulate the output from Initial
    {
        ItemsSketch<String> sketch = new ItemsSketch<String>(8);
        sketch.update("a");
        sketch.update("b");
        DataBag innerBag = PigUtil.tuplesToBag(PigUtil.objectsToTuple(new DataByteArray(sketch.toByteArray(new ArrayOfStringsSerDe()))));
        bag.add(PigUtil.objectsToTuple(innerBag));
    }
    // this is to simulate the output from a prior call of IntermediateFinal
    {
        ItemsSketch<String> sketch = new ItemsSketch<String>(8);
        sketch.update("a", 2L);
        sketch.update("b", 3L);
        bag.add(PigUtil.objectsToTuple(new DataByteArray(sketch.toByteArray(new ArrayOfStringsSerDe()))));
    }
    Tuple resultTuple = func.exec(PigUtil.objectsToTuple(bag));
    Assert.assertNotNull(resultTuple);
    Assert.assertEquals(resultTuple.size(), 1);
    DataByteArray bytes = (DataByteArray) resultTuple.get(0);
    Assert.assertTrue(bytes.size() > 0);
    ItemsSketch<String> sketch = ItemsSketch.getInstance(Memory.wrap(bytes.get()), new ArrayOfStringsSerDe());
    Assert.assertFalse(sketch.isEmpty());
    Assert.assertEquals(sketch.getNumActiveItems(), 2);
    Assert.assertEquals(sketch.getEstimate("a"), 3);
    Assert.assertEquals(sketch.getEstimate("b"), 4);
}

Example 10 with ItemsSketch

use of com.yahoo.sketches.frequencies.ItemsSketch in project sketches-pig by DataSketches.

the class UnionFrequentStringsSketchTest method exec.

@Test
public void exec() throws Exception {
    EvalFunc<Tuple> func = new UnionFrequentStringsSketch("8");
    DataBag bag = BagFactory.getInstance().newDefaultBag();
    {
        ItemsSketch<String> sketch = new ItemsSketch<String>(8);
        sketch.update("a");
        sketch.update("b");
        bag.add(PigUtil.objectsToTuple(new DataByteArray(sketch.toByteArray(new ArrayOfStringsSerDe()))));
    }
    {
        ItemsSketch<String> sketch = new ItemsSketch<String>(8);
        sketch.update("a");
        sketch.update("b");
        bag.add(PigUtil.objectsToTuple(new DataByteArray(sketch.toByteArray(new ArrayOfStringsSerDe()))));
    }
    Tuple resultTuple = func.exec(PigUtil.objectsToTuple(bag));
    Assert.assertNotNull(resultTuple);
    Assert.assertEquals(resultTuple.size(), 1);
    DataByteArray bytes = (DataByteArray) resultTuple.get(0);
    Assert.assertTrue(bytes.size() > 0);
    ItemsSketch<String> sketch = ItemsSketch.getInstance(Memory.wrap(bytes.get()), new ArrayOfStringsSerDe());
    Assert.assertEquals(sketch.getNumActiveItems(), 2);
    Assert.assertEquals(sketch.getEstimate("a"), 2);
    Assert.assertEquals(sketch.getEstimate("b"), 2);
}