Examples with GroupOperation com.yahoo.bullet.querying.aggregations.grouping.GroupOperation used on opensource projects

Example 11 with GroupOperation

use of com.yahoo.bullet.querying.aggregations.grouping.GroupOperation in project bullet-core by yahoo.

the class TupleSketchingStrategyTest method testMetadata.

@Test
public void testMetadata() {
    Map<String, String> fields = singletonMap("fieldA", "fieldA");
    // Nominal Entries is 32. Aggregation size is also 32
    TupleSketchingStrategy groupBy = makeGroupBy(makeConfiguration(32), fields, 32, singletonList(new GroupOperation(COUNT, null, "count")), ALL_METADATA);
    // Generate 4 batches of 64 records with 0 - 63 in fieldA.
    IntStream.range(0, 256).mapToObj(i -> RecordBox.get().add("fieldA", i % 64).getRecord()).forEach(groupBy::consume);
    Clip aggregate = groupBy.getResult();
    Assert.assertNotNull(aggregate);
    List<BulletRecord> records = aggregate.getRecords();
    Assert.assertEquals(records.size(), 32);
    records.forEach(r -> Assert.assertTrue(Integer.valueOf((String) r.typedGet("fieldA").getValue()) < 64));
    Map<String, Object> meta = aggregate.getMeta().asMap();
    Assert.assertEquals(meta.size(), 1);
    Map<String, Object> stats = (Map<String, Object>) meta.get("aggregate_stats");
    Assert.assertEquals(stats.size(), 4);
    Assert.assertTrue((Boolean) stats.get("isEstimate"));
    double theta = (Double) stats.get("theta");
    Assert.assertTrue(theta <= 1.0);
    double groupEstimate = (Double) stats.get("uniquesApprox");
    Assert.assertTrue(stats.containsKey("stddev"));
    Map<String, Map<String, Double>> standardDeviations = (Map<String, Map<String, Double>>) stats.get("stddev");
    Assert.assertEquals(standardDeviations.size(), 3);
    double upperOneSigma = standardDeviations.get(KMVSketch.META_STD_DEV_1).get(KMVSketch.META_STD_DEV_UB);
    double lowerOneSigma = standardDeviations.get(KMVSketch.META_STD_DEV_1).get(KMVSketch.META_STD_DEV_LB);
    double upperTwoSigma = standardDeviations.get(KMVSketch.META_STD_DEV_2).get(KMVSketch.META_STD_DEV_UB);
    double lowerTwoSigma = standardDeviations.get(KMVSketch.META_STD_DEV_2).get(KMVSketch.META_STD_DEV_LB);
    double upperThreeSigma = standardDeviations.get(KMVSketch.META_STD_DEV_3).get(KMVSketch.META_STD_DEV_UB);
    double lowerThreeSigma = standardDeviations.get(KMVSketch.META_STD_DEV_3).get(KMVSketch.META_STD_DEV_LB);
    Assert.assertTrue(groupEstimate >= lowerOneSigma);
    Assert.assertTrue(groupEstimate <= upperOneSigma);
    Assert.assertTrue(groupEstimate >= lowerTwoSigma);
    Assert.assertTrue(groupEstimate <= upperTwoSigma);
    Assert.assertTrue(groupEstimate >= lowerThreeSigma);
    Assert.assertTrue(groupEstimate <= upperThreeSigma);
    Assert.assertTrue(groupEstimate <= upperThreeSigma);
    Assert.assertEquals(groupBy.getRecords(), aggregate.getRecords());
    Assert.assertEquals(groupBy.getMetadata().asMap(), aggregate.getMeta().asMap());
}

Example 12 with GroupOperation

use of com.yahoo.bullet.querying.aggregations.grouping.GroupOperation in project bullet-core by yahoo.

the class QuerierTest method testOuterQuery.

@Test
public void testOuterQuery() {
    Expression outerQueryFilter = new BinaryExpression(new FieldExpression("count"), new ValueExpression(1), Operation.GREATER_THAN);
    Query outerQuery = new Query(new Projection(), outerQueryFilter, new Raw(3), null, new Window(), null);
    GroupBy groupBy = new GroupBy(null, singletonMap("color", "color"), singleton(new GroupOperation(GroupOperation.GroupOperationType.COUNT, null, "count")));
    Query query = new Query(null, new Projection(), null, groupBy, null, outerQuery, new Window(), null);
    Querier querier = make(Querier.Mode.ALL, query);
    BulletRecord red = RecordBox.get().add("color", "red").getRecord();
    BulletRecord orange = RecordBox.get().add("color", "orange").getRecord();
    BulletRecord yellow = RecordBox.get().add("color", "yellow").getRecord();
    BulletRecord green = RecordBox.get().add("color", "green").getRecord();
    BulletRecord blue = RecordBox.get().add("color", "blue").getRecord();
    BulletRecord indigo = RecordBox.get().add("color", "indigo").getRecord();
    BulletRecord violet = RecordBox.get().add("color", "violet").getRecord();
    querier.consume(red);
    querier.consume(orange);
    querier.consume(yellow);
    querier.consume(green);
    querier.consume(blue);
    querier.consume(indigo);
    querier.consume(violet);
    querier.consume(yellow);
    querier.consume(green);
    querier.consume(blue);
    querier.consume(indigo);
    querier.consume(violet);
    Clip result = querier.getResult();
    List<BulletRecord> records = result.getRecords();
    Assert.assertEquals(records.size(), 3);
    Assert.assertFalse(records.stream().anyMatch(record -> record.typedGet("color").getValue().equals("red") || record.typedGet("color").getValue().equals("orange")));
    Assert.assertTrue(records.stream().allMatch(record -> record.typedGet("count").getValue().equals(2L)));
    Map<String, Object> metadata = result.getMeta().asMap();
    Assert.assertTrue(metadata.containsKey("Inner Query"));
}

Example 13 with GroupOperation

use of com.yahoo.bullet.querying.aggregations.grouping.GroupOperation in project bullet-core by yahoo.

the class GroupAllStrategyTest method testMax.

@Test
public void testMax() {
    GroupAllStrategy strategy = makeGroupAll(new GroupOperation(GroupOperation.GroupOperationType.MAX, "someField", "max"));
    Assert.assertNotNull(strategy.getData());
    strategy.consume(RecordBox.get().addNull("someField").getRecord());
    BulletRecord expected = RecordBox.get().addNull("max").getRecord();
    Assert.assertEquals(strategy.getResult().getRecords().get(0), expected);
    strategy.consume(RecordBox.get().add("someField", -4.8).getRecord());
    strategy.consume(RecordBox.get().add("someField", -8.8).getRecord());
    strategy.consume(RecordBox.get().add("someField", 51.44).getRecord());
    expected = RecordBox.get().add("max", 51.44).getRecord();
    Assert.assertEquals(strategy.getResult().getRecords().get(0), expected);
    strategy.consume(RecordBox.get().addNull("someField").getRecord());
    expected = RecordBox.get().add("max", 51.44).getRecord();
    Assert.assertEquals(strategy.getResult().getRecords().get(0), expected);
    strategy.consume(RecordBox.get().add("someField", 88.0).getRecord());
    expected = RecordBox.get().add("max", 88.0).getRecord();
    Assert.assertEquals(strategy.getResult().getRecords().get(0), expected);
    Assert.assertEquals(strategy.getResult().getRecords().size(), 1);
    Assert.assertEquals(strategy.getRecords(), strategy.getResult().getRecords());
    Assert.assertEquals(strategy.getMetadata().asMap(), strategy.getResult().getMeta().asMap());
}

Example 14 with GroupOperation

use of com.yahoo.bullet.querying.aggregations.grouping.GroupOperation in project bullet-core by yahoo.

the class GroupAllStrategyTest method testSum.

@Test
public void testSum() {
    GroupAllStrategy strategy = makeGroupAll(new GroupOperation(GroupOperation.GroupOperationType.SUM, "someField", "sum"));
    Assert.assertNotNull(strategy.getData());
    strategy.consume(RecordBox.get().addNull("someField").getRecord());
    BulletRecord expected = RecordBox.get().addNull("sum").getRecord();
    Assert.assertEquals(strategy.getResult().getRecords().get(0), expected);
    strategy.consume(RecordBox.get().add("someField", -4.8).getRecord());
    strategy.consume(RecordBox.get().add("someField", -8).getRecord());
    strategy.consume(RecordBox.get().add("someField", 51.44).getRecord());
    expected = RecordBox.get().add("sum", 38.64).getRecord();
    Assert.assertEquals(strategy.getResult().getRecords().get(0), expected);
    strategy.consume(RecordBox.get().addNull("someField").getRecord());
    expected = RecordBox.get().add("sum", 38.64).getRecord();
    Assert.assertEquals(strategy.getResult().getRecords().get(0), expected);
    strategy.consume(RecordBox.get().add("someField", 88.0).getRecord());
    expected = RecordBox.get().add("sum", 126.64).getRecord();
    Assert.assertEquals(strategy.getResult().getRecords().get(0), expected);
    Assert.assertEquals(strategy.getResult().getRecords().size(), 1);
    Assert.assertEquals(strategy.getRecords(), strategy.getResult().getRecords());
    Assert.assertEquals(strategy.getMetadata().asMap(), strategy.getResult().getMeta().asMap());
}

Example 15 with GroupOperation

use of com.yahoo.bullet.querying.aggregations.grouping.GroupOperation in project bullet-core by yahoo.

the class GroupAllStrategyTest method testCombiningMetricsFail.

@Test
public void testCombiningMetricsFail() {
    GroupAllStrategy strategy = makeGroupAll(new GroupOperation(GroupOperation.GroupOperationType.COUNT, null, "count"));
    BulletRecord someRecord = RecordBox.get().add("foo", 1).getRecord();
    IntStream.range(0, 10).forEach(i -> strategy.consume(someRecord));
    // Not a serialized GroupData
    strategy.combine(String.valueOf(242).getBytes());
    Assert.assertNotNull(strategy.getData());
    List<BulletRecord> aggregate = strategy.getResult().getRecords();
    Assert.assertEquals(aggregate.size(), 1);
    BulletRecord actual = aggregate.get(0);
    // Unchanged count
    BulletRecord expected = RecordBox.get().add("count", 10L).getRecord();
    Assert.assertEquals(actual, expected);
    Assert.assertEquals(strategy.getRecords(), aggregate);
    Assert.assertEquals(strategy.getMetadata().asMap(), strategy.getResult().getMeta().asMap());
}