Examples with DataToSketch com.yahoo.sketches.pig.theta.DataToSketch used on opensource projects

Example 1 with DataToSketch

use of com.yahoo.sketches.pig.theta.DataToSketch in project sketches-pig by DataSketches.

the class DataToSketchTest method testAccumulate.

@Test
public void testAccumulate() throws IOException {
    Accumulator<Tuple> func = new DataToSketch("128");
    Tuple inputTuple = TupleFactory.getInstance().newTuple(1);
    DataBag bag = BagFactory.getInstance().newDefaultBag();
    inputTuple.set(0, bag);
    for (int ii = 0; ii < 64; ii++) {
        Tuple dataTuple = TupleFactory.getInstance().newTuple(1);
        dataTuple.set(0, ii);
        bag.add(dataTuple);
    }
    func.accumulate(inputTuple);
    inputTuple = TupleFactory.getInstance().newTuple(1);
    bag = BagFactory.getInstance().newDefaultBag();
    inputTuple.set(0, bag);
    for (int ii = 0; ii < 27; ii++) {
        Tuple dataTuple = TupleFactory.getInstance().newTuple(1);
        dataTuple.set(0, 64 + ii);
        bag.add(dataTuple);
    }
    func.accumulate(inputTuple);
    Tuple resultTuple = func.getValue();
    assertNotNull(resultTuple);
    assertEquals(resultTuple.size(), 1);
    DataByteArray bytes = (DataByteArray) resultTuple.get(0);
    assertTrue(bytes.size() > 0);
    Sketch sketch = tupleToSketch(resultTuple, seed_);
    assertEquals(sketch.getEstimate(), 91.0, 0.0);
    // after cleanup, the value should always be 0
    func.cleanup();
    resultTuple = func.getValue();
    assertNotNull(resultTuple);
    assertEquals(resultTuple.size(), 1);
    bytes = (DataByteArray) resultTuple.get(0);
    assertTrue(bytes.size() > 0);
    sketch = tupleToSketch(resultTuple, seed_);
    assertEquals(sketch.getEstimate(), 0.0, 0.0);
}

Example 2 with DataToSketch

use of com.yahoo.sketches.pig.theta.DataToSketch in project sketches-pig by DataSketches.

the class DataToSketchTest method textTopExec2.

/*
   * DataToSketch <br>
   * Tests all possible data types: NULL, BYTE, INTEGER, LONG, FLOAT, DOUBLE,
   * BYTEARRAY, CHARARRAY. Tests rejection of a non-simple type.
   */
// still triggers unchecked warning
@SuppressWarnings("unchecked")
@Test
public void textTopExec2() throws IOException {
    TupleFactory tupleFactory = TupleFactory.getInstance();
    BagFactory bagFactory = BagFactory.getInstance();
    String[] ctorArgs = { "128" };
    EvalFunc<Tuple> dataUdf = (EvalFunc<Tuple>) PigContext.instantiateFuncFromSpec(new FuncSpec(udfName, ctorArgs));
    // EvalFunc<Tuple> resultUdf = (EvalFunc<Tuple>)PigContext.
    // instantiateFuncFromSpec(new FuncSpec(resultUdfName));
    Tuple t;
    DataBag bag = bagFactory.newDefaultBag();
    // empty with a null
    bag.add(tupleFactory.newTuple());
    // 1 empty field
    bag.add(tupleFactory.newTuple(1));
    // 1
    t = tupleFactory.newTuple(1);
    t.set(0, new Byte((byte) 1));
    bag.add(t);
    // 2
    t = tupleFactory.newTuple(1);
    // int
    t.set(0, new Integer(2));
    bag.add(t);
    // 3
    t = tupleFactory.newTuple(1);
    t.set(0, new Long(3));
    bag.add(t);
    // 4
    t = tupleFactory.newTuple(1);
    t.set(0, new Float(4));
    bag.add(t);
    // 5
    t = tupleFactory.newTuple(1);
    t.set(0, new Double(5));
    bag.add(t);
    // 6
    t = tupleFactory.newTuple(1);
    byte[] bArr = { 1, 2, 3 };
    t.set(0, new DataByteArray(bArr));
    bag.add(t);
    // -ignore
    t = tupleFactory.newTuple(1);
    // empty
    byte[] bArr2 = new byte[0];
    t.set(0, new DataByteArray(bArr2));
    bag.add(t);
    // 7
    t = tupleFactory.newTuple(1);
    t.set(0, new Double(-0.0));
    bag.add(t);
    // 7 duplicate
    t = tupleFactory.newTuple(1);
    t.set(0, new Double(0.0));
    bag.add(t);
    // 8
    t = tupleFactory.newTuple(1);
    String s = "abcde";
    t.set(0, s);
    bag.add(t);
    // - ignore
    t = tupleFactory.newTuple(1);
    // empty
    String s2 = "";
    t.set(0, s2);
    bag.add(t);
    Tuple in = tupleFactory.newTuple(1);
    in.set(0, bag);
    // should return a sketch
    Tuple resultTuple = dataUdf.exec(in);
    assertNotNull(resultTuple);
    assertEquals(resultTuple.size(), 1);
    DataByteArray bytes = (DataByteArray) resultTuple.get(0);
    assertTrue(bytes.size() > 0);
    Sketch sketch = tupleToSketch(resultTuple, seed_);
    assertEquals(sketch.getEstimate(), 8.0, 0.0);
}

Example 3 with DataToSketch

use of com.yahoo.sketches.pig.theta.DataToSketch in project sketches-pig by DataSketches.

the class DataToSketchTest method checkConstructors.

@SuppressWarnings("unused")
@Test
public void checkConstructors() {
    DataToSketch inter = new DataToSketch();
    inter = new DataToSketch("1024");
    inter = new DataToSketch("1024", "1.0");
    inter = new DataToSketch("1024", "1.0", "9001");
    inter = new DataToSketch(1024, (float) 1.0, 9001);
    DataToSketch.Initial initial = new DataToSketch.Initial();
    initial = new DataToSketch.Initial("1024");
    initial = new DataToSketch.Initial("1024", "1.0");
    initial = new DataToSketch.Initial("1024", "1.0", "9001");
    DataToSketch.IntermediateFinal interFin = new DataToSketch.IntermediateFinal();
    interFin = new DataToSketch.IntermediateFinal("1024");
    interFin = new DataToSketch.IntermediateFinal("1024", "1.0");
    interFin = new DataToSketch.IntermediateFinal("1024", "1.0", "9001");
    interFin = new DataToSketch.IntermediateFinal(1024, (float) 1.0, 9001);
}

Example 4 with DataToSketch

use of com.yahoo.sketches.pig.theta.DataToSketch in project sketches-pig by DataSketches.

the class DataToSketchTest method checkNotDBAExcep.

@Test
public void checkNotDBAExcep() throws IOException {
    DataToSketch inter = new DataToSketch();
    // create inputTuple and a bag, add bag to inputTuple
    Tuple inputTuple = TupleFactory.getInstance().newTuple(1);
    DataBag bag = BagFactory.getInstance().newDefaultBag();
    inputTuple.set(0, bag);
    Tuple innerTuple = TupleFactory.getInstance().newTuple(1);
    bag.add(innerTuple);
    // add empty tuple
    inter.accumulate(inputTuple);
    // not a DBA
    innerTuple.set(0, new Double(1.0));
    inter = new DataToSketch();
    // add wrong type
    inter.accumulate(inputTuple);
}

Example 5 with DataToSketch

use of com.yahoo.sketches.pig.theta.DataToSketch in project sketches-pig by DataSketches.

the class DataToSketchTest method checkMisc.

@Test
@SuppressWarnings("unused")
public void checkMisc() throws IOException {
    DataToSketch dts = new DataToSketch("512", "1.0");
    dts = new DataToSketch("512", "1.0", "9001");
    DataToSketch.Initial dtsi = new DataToSketch.Initial("512", "1.0");
    DataToSketch.IntermediateFinal dtsif = new DataToSketch.IntermediateFinal("512", "1.0");
    // null bag
    Tuple inputTuple = TupleFactory.getInstance().newTuple(1);
    dts.accumulate(inputTuple);
    Tuple resultTuple = dts.getValue();
    Sketch sketch = tupleToSketch(resultTuple, seed_);
    assertTrue(sketch.isEmpty());
}