use of com.yahoo.sketches.hll.HllSketch in project Gaffer by gchq.
the class HllSketchKryoSerializerTest method getTestObject.
@Override
public HllSketch getTestObject() {
final HllSketch sketch = new HllSketch(15);
sketch.update("A");
sketch.update("B");
sketch.update("C");
return sketch;
}
use of com.yahoo.sketches.hll.HllSketch in project Gaffer by gchq.
the class HllSketchJsonDeserialiser method deserialize.
@Override
public HllSketch deserialize(final JsonParser jsonParser, final DeserializationContext deserializationContext) throws IOException, JsonProcessingException {
final TreeNode treeNode = jsonParser.getCodec().readTree(jsonParser);
final HllSketch hllp;
final TextNode jsonNodes = (TextNode) treeNode.get(HllSketchJsonConstants.BYTES);
if (isNull(jsonNodes)) {
final IntNode kNode = (IntNode) treeNode.get(HllSketchJsonConstants.LOG_K);
final int k = nonNull(kNode) ? kNode.asInt(DEFAULT_LOG_K) : DEFAULT_LOG_K;
hllp = new HllSketch(k);
} else {
hllp = HllSketch.heapify(jsonNodes.binaryValue());
}
final ArrayNode offers = (ArrayNode) treeNode.get(VALUES);
if (nonNull(offers)) {
for (final JsonNode offer : offers) {
if (nonNull(offer)) {
hllp.update(offer.asText());
}
}
}
return hllp;
}
use of com.yahoo.sketches.hll.HllSketch in project gaffer-doc by gchq.
the class HllSketchWalkthrough method run.
@Override
public CloseableIterable<? extends Element> run() throws OperationException {
// / [graph] create a graph using our schema and store properties
// ---------------------------------------------------------
final Graph graph = new Graph.Builder().config(getDefaultGraphConfig()).addSchemas(StreamUtil.openStreams(getClass(), schemaPath)).storeProperties(getDefaultStoreProperties()).build();
// ---------------------------------------------------------
// [user] Create a user
// ---------------------------------------------------------
final User user = new User("user01");
// ---------------------------------------------------------
// [add] addElements - add the edges to the graph
// ---------------------------------------------------------
final Set<String> dummyData = Collections.singleton("");
final OperationChain<Void> addOpChain = new OperationChain.Builder().first(new GenerateElements.Builder<String>().generator(new HllSketchElementGenerator()).input(dummyData).build()).then(new AddElements()).build();
graph.execute(addOpChain, user);
// ---------------------------------------------------------
print("Added 1000 entities for vertex A, each time with a HllSketch containing a vertex that A was seen in an edge with");
// [get] Get all entities
// ---------------------------------------------------------
CloseableIterable<? extends Element> allEntities = graph.execute(new GetAllElements(), user);
// ---------------------------------------------------------
print("\nAll edges:");
for (final Element entity : allEntities) {
print("GET_ALL_ENTITIES_RESULT", entity.toString());
}
// [get the approximate degree of a] Get the entity for A and print out the estimate of the degree
// ---------------------------------------------------------
final GetElements query = new GetElements.Builder().input(new EntitySeed("A")).build();
final Element element;
try (final CloseableIterable<? extends Element> elements = graph.execute(query, user)) {
element = elements.iterator().next();
}
final HllSketch hllSketch = (HllSketch) element.getProperty("approxCardinality");
final double approxDegree = hllSketch.getEstimate();
final String degreeEstimate = "Entity A has approximate degree " + approxDegree;
// ---------------------------------------------------------
print("\nEntity A with an estimate of its degree");
print("GET_APPROX_DEGREE_FOR_ENTITY_A", degreeEstimate);
return null;
}
use of com.yahoo.sketches.hll.HllSketch in project sketches-pig by DataSketches.
the class DataToSketchTest method execEmptyBag.
@Test
public void execEmptyBag() throws Exception {
EvalFunc<DataByteArray> func = new DataToSketch("10", "HLL_6");
DataByteArray result = func.exec(tupleFactory.newTuple(bagFactory.newDefaultBag()));
HllSketch sketch = getSketch(result);
Assert.assertTrue(sketch.isEmpty());
Assert.assertEquals(sketch.getLgConfigK(), 10);
Assert.assertEquals(sketch.getTgtHllType(), TgtHllType.HLL_6);
}
use of com.yahoo.sketches.hll.HllSketch in project sketches-pig by DataSketches.
the class DataToSketchTest method algebraicIntermediateFromIntermediate.
@Test
public void algebraicIntermediateFromIntermediate() throws Exception {
@SuppressWarnings("unchecked") EvalFunc<Tuple> func = (EvalFunc<Tuple>) Class.forName(new DataToSketch().getIntermed()).newInstance();
HllSketch inputSketch = new HllSketch(12);
inputSketch.update("a");
inputSketch.update("b");
DataBag bag = bagFactory.newDefaultBag();
bag.add(tupleFactory.newTuple(new DataByteArray(inputSketch.toCompactByteArray())));
Tuple result = func.exec(tupleFactory.newTuple(bag));
HllSketch sketch = getSketch((DataByteArray) result.get(0));
Assert.assertFalse(sketch.isEmpty());
Assert.assertEquals(sketch.getEstimate(), 2.0, 0.01);
}
Aggregations