Examples with ArrayOfStringsSerDe com.yahoo.sketches.ArrayOfStringsSerDe used on opensource projects

Example 21 with ArrayOfStringsSerDe

use of com.yahoo.sketches.ArrayOfStringsSerDe in project sketches-pig by DataSketches.

the class UnionFrequentStringsSketchTest method accumulatorEmptyInnerTuple.

@Test
public void accumulatorEmptyInnerTuple() throws Exception {
    Accumulator<Tuple> func = new UnionFrequentStringsSketch("8");
    func.accumulate(PigUtil.objectsToTuple(PigUtil.tuplesToBag(TupleFactory.getInstance().newTuple())));
    Tuple resultTuple = func.getValue();
    Assert.assertNotNull(resultTuple);
    Assert.assertEquals(resultTuple.size(), 1);
    DataByteArray bytes = (DataByteArray) resultTuple.get(0);
    Assert.assertTrue(bytes.size() > 0);
    ItemsSketch<String> sketch = ItemsSketch.getInstance(Memory.wrap(bytes.get()), new ArrayOfStringsSerDe());
    Assert.assertTrue(sketch.isEmpty());
    Assert.assertEquals(sketch.getNumActiveItems(), 0);
}

Example 22 with ArrayOfStringsSerDe

use of com.yahoo.sketches.ArrayOfStringsSerDe in project sketches-pig by DataSketches.

the class UnionFrequentStringsSketchTest method accumulatorEmptySketch.

@Test
public void accumulatorEmptySketch() throws Exception {
    Accumulator<Tuple> func = new UnionFrequentStringsSketch("8");
    DataBag bag = BagFactory.getInstance().newDefaultBag();
    {
        ItemsSketch<String> sketch = new ItemsSketch<String>(8);
        bag.add(PigUtil.objectsToTuple(new DataByteArray(sketch.toByteArray(new ArrayOfStringsSerDe()))));
    }
    func.accumulate(PigUtil.objectsToTuple(bag));
    Tuple resultTuple = func.getValue();
    Assert.assertNotNull(resultTuple);
    Assert.assertEquals(resultTuple.size(), 1);
    DataByteArray bytes = (DataByteArray) resultTuple.get(0);
    Assert.assertTrue(bytes.size() > 0);
    ItemsSketch<String> sketch = ItemsSketch.getInstance(Memory.wrap(bytes.get()), new ArrayOfStringsSerDe());
    Assert.assertTrue(sketch.isEmpty());
    Assert.assertEquals(sketch.getNumActiveItems(), 0);
}

Example 23 with ArrayOfStringsSerDe

use of com.yahoo.sketches.ArrayOfStringsSerDe in project sketches-pig by DataSketches.

the class GetQuantilesFromStringsSketch method exec.

@Override
public Tuple exec(final Tuple input) throws IOException {
    if (input.size() < 2) {
        throw new IllegalArgumentException("expected two or more inputs: sketch and list of fractions");
    }
    if (!(input.get(0) instanceof DataByteArray)) {
        throw new IllegalArgumentException("expected a DataByteArray as a sketch, got " + input.get(0).getClass().getSimpleName());
    }
    final DataByteArray dba = (DataByteArray) input.get(0);
    final ItemsSketch<String> sketch = ItemsSketch.getInstance(Memory.wrap(dba.get()), Comparator.naturalOrder(), new ArrayOfStringsSerDe());
    if (sketch.isEmpty()) {
        return null;
    }
    if (input.size() == 2) {
        final Object arg = input.get(1);
        if (arg instanceof Integer) {
            // number of evenly spaced intervals
            return TupleFactory.getInstance().newTuple(Arrays.asList(sketch.getQuantiles((int) arg)));
        } else if (arg instanceof Double) {
            // just one fraction
            return TupleFactory.getInstance().newTuple(Arrays.asList(sketch.getQuantile((double) arg)));
        } else {
            throw new IllegalArgumentException("expected a double value as a fraction or an integer value" + " as a number of evenly spaced intervals, got " + arg.getClass().getSimpleName());
        }
    }
    // more than one number - must be double fractions
    final double[] fractions = new double[input.size() - 1];
    for (int i = 1; i < input.size(); i++) {
        if (!(input.get(i) instanceof Double)) {
            throw new IllegalArgumentException("expected a double value as a fraction, got " + input.get(i).getClass().getSimpleName());
        }
        fractions[i - 1] = (double) input.get(i);
    }
    return TupleFactory.getInstance().newTuple(Arrays.asList(sketch.getQuantiles(fractions)));
}

Example 24 with ArrayOfStringsSerDe

use of com.yahoo.sketches.ArrayOfStringsSerDe in project sketches-core by DataSketches.

the class ReservoirItemsUnionTest method checkBadSerVer.

@Test(expectedExceptions = SketchesArgumentException.class)
public void checkBadSerVer() {
    final ReservoirItemsUnion<String> riu = ReservoirItemsUnion.newInstance(1024);
    final WritableMemory mem = WritableMemory.wrap(riu.toByteArray(new ArrayOfStringsSerDe()));
    // corrupt the serialization version
    mem.putByte(SER_VER_BYTE, (byte) 0);
    ReservoirItemsUnion.heapify(mem, new ArrayOfStringsSerDe());
    fail();
}

Example 25 with ArrayOfStringsSerDe

use of com.yahoo.sketches.ArrayOfStringsSerDe in project sketches-core by DataSketches.

the class VarOptItemsUnionTest method serializeEmptyUnion.

@Test
public void serializeEmptyUnion() {
    final int k = 100;
    final VarOptItemsUnion<String> union = VarOptItemsUnion.newInstance(k);
    // null inputs to update() should leave the union empty
    union.update(null);
    union.update(null, new ArrayOfStringsSerDe());
    final ArrayOfStringsSerDe serDe = new ArrayOfStringsSerDe();
    final byte[] bytes = union.toByteArray(serDe);
    assertEquals(bytes.length, 8);
    final Memory mem = Memory.wrap(bytes);
    final VarOptItemsUnion<String> rebuilt = VarOptItemsUnion.heapify(mem, serDe);
    final VarOptItemsSketch<String> sketch = rebuilt.getResult();
    assertEquals(sketch.getN(), 0);
    assertEquals(rebuilt.toString(), union.toString());
}