Examples with HyperLogLogPlus com.clearspring.analytics.stream.cardinality.HyperLogLogPlus used on opensource projects

Example 6 with HyperLogLogPlus

use of com.clearspring.analytics.stream.cardinality.HyperLogLogPlus in project gaffer-doc by gchq.

the class Cardinalities method run.

public CloseableIterable<? extends Element> run() throws OperationException, IOException {
    // [graph] create a graph using our schema and store properties
    // ---------------------------------------------------------
    final Graph graph = new Graph.Builder().config(getDefaultGraphConfig()).addSchemas(StreamUtil.openStreams(getClass(), schemaPath)).storeProperties(getDefaultStoreProperties()).build();
    // ---------------------------------------------------------
    // [user] Create a user
    // ---------------------------------------------------------
    final User user = new User("user01");
    // ---------------------------------------------------------
    // [add] Create a data generator and add the edges to the graph using an operation chain consisting of:
    // generateElements - generating edges from the data (note these are directed edges)
    // addElements - add the edges to the graph
    // ---------------------------------------------------------
    final OperationChain<Void> addOpChain = new OperationChain.Builder().first(new GenerateElements.Builder<String>().generator(new RoadAndRoadUseWithTimesAndCardinalitiesElementGenerator()).input(IOUtils.readLines(StreamUtil.openStream(getClass(), dataPath))).build()).then(new AddElements()).build();
    graph.execute(addOpChain, user);
    // ---------------------------------------------------------
    print("The elements have been added.");
    // [get] Get all edges
    // ---------------------------------------------------------
    final CloseableIterable<? extends Element> edges = graph.execute(new GetAllElements(), user);
    // ---------------------------------------------------------
    print("\nAll edges:");
    for (final Element edge : edges) {
        print("GET_ALL_EDGES_RESULT", edge.toString());
    }
    // [get all cardinalities] Get all cardinalities
    // ---------------------------------------------------------
    final GetAllElements getAllCardinalities = new GetAllElements.Builder().view(new View.Builder().entity("Cardinality").build()).build();
    // ---------------------------------------------------------
    final CloseableIterable<? extends Element> allCardinalities = graph.execute(getAllCardinalities, user);
    print("\nAll cardinalities");
    for (final Element cardinality : allCardinalities) {
        final String edgeGroup = cardinality.getProperty("edgeGroup").toString();
        print("ALL_CARDINALITIES_RESULT", "Vertex " + ((Entity) cardinality).getVertex() + " " + edgeGroup + ": " + ((HyperLogLogPlus) cardinality.getProperty("hllp")).cardinality());
    }
    // [get all summarised cardinalities] Get all summarised cardinalities over all edges
    // ---------------------------------------------------------
    final GetAllElements getAllSummarisedCardinalities = new GetAllElements.Builder().view(new View.Builder().entity("Cardinality", new ViewElementDefinition.Builder().groupBy().build()).build()).build();
    // ---------------------------------------------------------
    final CloseableIterable<? extends Element> allSummarisedCardinalities = graph.execute(getAllSummarisedCardinalities, user);
    print("\nAll summarised cardinalities");
    for (final Element cardinality : allSummarisedCardinalities) {
        final String edgeGroup = cardinality.getProperty("edgeGroup").toString();
        print("ALL_SUMMARISED_CARDINALITIES_RESULT", "Vertex " + ((Entity) cardinality).getVertex() + " " + edgeGroup + ": " + ((HyperLogLogPlus) cardinality.getProperty("hllp")).cardinality());
    }
    // [get roaduse edge cardinality 10] Get the cardinality value at vertex 10 for RoadUse edges
    // ---------------------------------------------------------
    final GetElements getCardinalities = new GetElements.Builder().input(new EntitySeed("10")).view(new View.Builder().entity("Cardinality", new ViewElementDefinition.Builder().preAggregationFilter(new ElementFilter.Builder().select("edgeGroup").execute(new IsEqual(CollectionUtil.treeSet("RoadUse"))).build()).build()).build()).build();
    // ---------------------------------------------------------
    final Element roadUse10Cardinality;
    try (final CloseableIterable<? extends Element> elements = graph.execute(getCardinalities, user)) {
        roadUse10Cardinality = elements.iterator().next();
    }
    print("\nRoadUse edge cardinality at vertex 10:");
    final String edgeGroup = (roadUse10Cardinality.getProperty("edgeGroup")).toString();
    print("CARDINALITY_OF_10_RESULT", "Vertex " + ((Entity) roadUse10Cardinality).getVertex() + " " + edgeGroup + ": " + ((HyperLogLogPlus) roadUse10Cardinality.getProperty("hllp")).cardinality());
    // [get 2 hops with a cardinality filter] 2 hops with a cardinality filter
    // ---------------------------------------------------------
    final OperationChain<CloseableIterable<? extends Element>> twoHopsWithCardinalityFilter = new OperationChain.Builder().first(new GetElements.Builder().input(new EntitySeed("M5")).view(new View.Builder().edge("RoadHasJunction").build()).build()).then(new GetElements.Builder().view(new View.Builder().entity("Cardinality", new ViewElementDefinition.Builder().preAggregationFilter(new ElementFilter.Builder().select("edgeGroup").execute(new IsEqual(CollectionUtil.treeSet("RoadUse"))).build()).groupBy().postAggregationFilter(new ElementFilter.Builder().select("hllp").execute(new HyperLogLogPlusIsLessThan(5)).build()).build()).build()).build()).then(new GetElements.Builder().view(new View.Builder().edge("RoadUse").build()).build()).build();
    // ---------------------------------------------------------
    final CloseableIterable<? extends Element> twoHopsWithCardinalityFilterResult = graph.execute(twoHopsWithCardinalityFilter, user);
    print("\n2 hops with cardinality filter result:");
    for (final Element element : twoHopsWithCardinalityFilterResult) {
        print("2_HOP_RESULT", element.toString());
    }
    return allSummarisedCardinalities;
}

Example 7 with HyperLogLogPlus

use of com.clearspring.analytics.stream.cardinality.HyperLogLogPlus in project gaffer-doc by gchq.

the class ElementWithVaryingGroupsGenerator method createCardinality.

private Entity createCardinality(final Object source, final Object destination, final Edge edge) {
    final HyperLogLogPlus hllp = new HyperLogLogPlus(5, 5);
    hllp.offer(destination);
    return new Entity.Builder().vertex(source).group("cardinality").property("edgeGroup", CollectionUtil.treeSet(edge.getGroup())).property("hllp", hllp).property("count", 1).build();
}

Example 8 with HyperLogLogPlus

use of com.clearspring.analytics.stream.cardinality.HyperLogLogPlus in project gaffer-doc by gchq.

the class RoadAndRoadUseWithTimesAndCardinalitiesElementGenerator method createCardinality.

private Entity createCardinality(final Object source, final Object destination, final Edge edge) {
    final HyperLogLogPlus hllp = new HyperLogLogPlus(5, 5);
    hllp.offer(destination);
    return new Entity.Builder().vertex(source).group("Cardinality").property("edgeGroup", CollectionUtil.treeSet(edge.getGroup())).property("hllp", hllp).property("count", 1L).build();
}

Example 9 with HyperLogLogPlus

use of com.clearspring.analytics.stream.cardinality.HyperLogLogPlus in project stream-lib by addthis.

the class ObyCount method main.

public static void main(String[] args) throws IOException {
    long updateRate = -1;
    long count = 0;
    if (args.length > 0) {
        try {
            updateRate = Long.parseLong(args[0]);
        } catch (NumberFormatException e) {
            System.err.print("Bad update rate: '" + args[0] + "'  Update rate must be an integer.");
            usage();
        }
    }
    BufferedReader in = new BufferedReader(new InputStreamReader(System.in));
    HyperLogLogPlus card = new HyperLogLogPlus(14, 25);
    String line = null;
    while ((line = in.readLine()) != null) {
        card.offer(line);
        count++;
        if (updateRate > 0 && count % updateRate == 0) {
            System.out.println(formatSummary(count, card.cardinality()));
        }
    }
    System.out.println(formatSummary(count, card.cardinality()));
}

Example 10 with HyperLogLogPlus

use of com.clearspring.analytics.stream.cardinality.HyperLogLogPlus in project Gaffer by gchq.

the class HyperLogLogPlusSerialiserTest method getExampleOutput.

@Override
protected HyperLogLogPlus getExampleOutput() {
    final HyperLogLogPlus hyperLogLogPlus1 = new HyperLogLogPlus(5, 5);
    hyperLogLogPlus1.offer("A");
    hyperLogLogPlus1.offer("B");
    return hyperLogLogPlus1;
}