Examples with Intersection com.yahoo.sketches.theta.Intersection used on opensource projects

Example 1 with Intersection

use of com.yahoo.sketches.theta.Intersection in project druid by druid-io.

the class SketchHolder method sketchSetOperation.

public static SketchHolder sketchSetOperation(Func func, int sketchSize, Object... holders) {
    //the final stages of query processing, ordered sketch would be of no use.
    switch(func) {
        case UNION:
            Union union = (Union) SetOperation.builder().build(sketchSize, Family.UNION);
            for (Object o : holders) {
                ((SketchHolder) o).updateUnion(union);
            }
            return SketchHolder.of(union);
        case INTERSECT:
            Intersection intersection = (Intersection) SetOperation.builder().build(sketchSize, Family.INTERSECTION);
            for (Object o : holders) {
                intersection.update(((SketchHolder) o).getSketch());
            }
            return SketchHolder.of(intersection.getResult(false, null));
        case NOT:
            if (holders.length < 1) {
                throw new IllegalArgumentException("A-Not-B requires atleast 1 sketch");
            }
            if (holders.length == 1) {
                return (SketchHolder) holders[0];
            }
            Sketch result = ((SketchHolder) holders[0]).getSketch();
            for (int i = 1; i < holders.length; i++) {
                AnotB anotb = (AnotB) SetOperation.builder().build(sketchSize, Family.A_NOT_B);
                anotb.update(result, ((SketchHolder) holders[i]).getSketch());
                result = anotb.getResult(false, null);
            }
            return SketchHolder.of(result);
        default:
            throw new IllegalArgumentException("Unknown sketch operation " + func);
    }
}

Example 2 with Intersection

use of com.yahoo.sketches.theta.Intersection in project sketches-core by DataSketches.

the class BoundsOnRatiosInThetaSketchedSetsTest method checkNormalReturns.

@Test
public void checkNormalReturns() {
    //4K
    UpdateSketch skA = Sketches.updateSketchBuilder().build();
    UpdateSketch skC = Sketches.updateSketchBuilder().build();
    int uA = 10000;
    int uC = 100000;
    for (int i = 0; i < uA; i++) {
        skA.update(i);
    }
    for (int i = 0; i < uC; i++) {
        skC.update(i + uA / 2);
    }
    Intersection inter = Sketches.setOperationBuilder().buildIntersection();
    inter.update(skA);
    inter.update(skC);
    CompactSketch skB = inter.getResult();
    double est = BoundsOnRatiosInThetaSketchedSets.getEstimateOfBoverA(skA, skB);
    double lb = BoundsOnRatiosInThetaSketchedSets.getLowerBoundForBoverA(skA, skB);
    double ub = BoundsOnRatiosInThetaSketchedSets.getUpperBoundForBoverA(skA, skB);
    assertTrue(ub > est);
    assertTrue(est > lb);
    assertEquals(est, 0.5, .03);
    println("ub : " + ub);
    println("est: " + est);
    println("lb : " + lb);
    //skA is now empty
    skA.reset();
    est = BoundsOnRatiosInThetaSketchedSets.getEstimateOfBoverA(skA, skB);
    lb = BoundsOnRatiosInThetaSketchedSets.getLowerBoundForBoverA(skA, skB);
    ub = BoundsOnRatiosInThetaSketchedSets.getUpperBoundForBoverA(skA, skB);
    println("ub : " + ub);
    println("est: " + est);
    println("lb : " + lb);
    //Now both are empty
    skC.reset();
    est = BoundsOnRatiosInThetaSketchedSets.getEstimateOfBoverA(skA, skC);
    lb = BoundsOnRatiosInThetaSketchedSets.getLowerBoundForBoverA(skA, skC);
    ub = BoundsOnRatiosInThetaSketchedSets.getUpperBoundForBoverA(skA, skC);
    println("ub : " + ub);
    println("est: " + est);
    println("lb : " + lb);
}

Example 3 with Intersection

use of com.yahoo.sketches.theta.Intersection in project Gaffer by gchq.

the class LoadAndQuery13 method run.

public Iterable<Entity> run() throws OperationException {
    // [user] Create a user
    // ---------------------------------------------------------
    final User user = new User("user01");
    // ---------------------------------------------------------
    // [graph] create a graph using our schema and store properties
    // ---------------------------------------------------------
    final Graph graph = new Graph.Builder().addSchemas(getSchemas()).storeProperties(getStoreProperties()).build();
    // ---------------------------------------------------------
    // [add] add the edges to the graph
    // ---------------------------------------------------------
    final Set<String> dummyData = Collections.singleton("");
    final OperationChain addOpChain = new OperationChain.Builder().first(new GenerateElements.Builder<String>().generator(new DataGenerator13()).objects(dummyData).build()).then(new AddElements()).build();
    graph.execute(addOpChain, user);
    // ---------------------------------------------------------
    log("Added 1000 edges A-B0, A-B1, ..., A-B999 on 10/1/17. For each edge we create an Entity with a union sketch" + " containing a string of the source and destination from the edge. Added 500 edges A-B750, A-B751, " + "..., A-B1249 for day 11/1/17. Again for each edge we create an Entity with a union sketch.");
    // [get entities] Get the entities for separate days
    // ---------------------------------------------------------
    final GetAllEntities get = new GetAllEntities();
    final Iterable<Entity> entities = graph.execute(get, user);
    for (final Entity entity : entities) {
        log("GET_ENTITIES", entity.toString());
    }
    // ---------------------------------------------------------
    // [get estimate separate days] Get the estimates out of the sketches for the separate days
    // ---------------------------------------------------------
    final GetAllEntities getAllEntities2 = new GetAllEntities();
    final Iterable<Entity> allEntities2 = graph.execute(getAllEntities2, user);
    final Iterator<Entity> it = allEntities2.iterator();
    final Entity entityDay1 = it.next();
    final CompactSketch sketchDay1 = ((Union) entityDay1.getProperty("size")).getResult();
    final Entity entityDay2 = it.next();
    final CompactSketch sketchDay2 = ((Union) entityDay2.getProperty("size")).getResult();
    final double estimateDay1 = sketchDay1.getEstimate();
    final double estimateDay2 = sketchDay2.getEstimate();
    // ---------------------------------------------------------
    log("\nThe estimates for the different days");
    log("GET_ESTIMATE_OVER_SEPARATE_DAYS", "" + estimateDay1);
    log("GET_ESTIMATE_OVER_SEPARATE_DAYS", "" + estimateDay2);
    // [get intersection] Get the number of edges in common across the two days
    // ---------------------------------------------------------
    final Intersection intersection = Sketches.setOperationBuilder().buildIntersection();
    intersection.update(sketchDay1);
    intersection.update(sketchDay2);
    final double intersectionSizeEstimate = intersection.getResult().getEstimate();
    // ---------------------------------------------------------
    log("\nThe estimate of the number of edges in common across the different days");
    log("PRINT_ESTIMATE", "" + intersectionSizeEstimate);
    // [get union across all days] Get the total number edges across the two days
    // ---------------------------------------------------------
    final GetAllEntities getAllEntities = new GetAllEntities.Builder().view(new View.Builder().entity("size", new ViewElementDefinition.Builder().groupBy().build()).build()).build();
    final Iterable<Entity> allEntities = graph.execute(getAllEntities, user);
    final Entity entity = allEntities.iterator().next();
    final double unionSizeEstimate = ((Union) entity.getProperty("size")).getResult().getEstimate();
    // ---------------------------------------------------------
    log("\nThe estimate of the number of edges across the different days");
    log("UNION_ESTIMATE", "" + unionSizeEstimate);
    return null;
}