Examples with COUNT com.yahoo.bullet.aggregations.grouping.GroupOperation.GroupOperationType.COUNT used on opensource projects

Example 6 with COUNT

use of com.yahoo.bullet.aggregations.grouping.GroupOperation.GroupOperationType.COUNT in project bullet-storm by yahoo.

the class JoinBoltTest method testTopK.

@Test
public void testTopK() {
    BulletConfig bulletConfig = TopKTest.makeConfiguration(ErrorType.NO_FALSE_NEGATIVES, 16);
    Map<String, String> fields = new HashMap<>();
    fields.put("A", "");
    fields.put("B", "foo");
    TopK topK = TopKTest.makeTopK(bulletConfig, makeAttributes(null, 5L), fields, 2, null);
    IntStream.range(0, 32).mapToObj(i -> RecordBox.get().add("A", i % 8).getRecord()).forEach(topK::consume);
    byte[] first = topK.getData();
    topK = TopKTest.makeTopK(bulletConfig, makeAttributes(null, 5L), fields, 2, null);
    IntStream.range(0, 8).mapToObj(i -> RecordBox.get().add("A", i % 2).getRecord()).forEach(topK::consume);
    byte[] second = topK.getData();
    bolt = new DonableJoinBolt(config, 2, true);
    setup(bolt);
    String aggregationQuery = makeAggregationQuery(TOP_K, 2, 5L, "cnt", Pair.of("A", ""), Pair.of("B", "foo"));
    Tuple query = TupleUtils.makeIDTuple(TupleClassifier.Type.QUERY_TUPLE, "42", aggregationQuery, EMPTY);
    bolt.execute(query);
    sendRawByteTuplesTo(bolt, "42", asList(first, second));
    BulletRecord expectedA = RecordBox.get().add("A", "0").add("foo", "null").add("cnt", 8L).getRecord();
    BulletRecord expectedB = RecordBox.get().add("A", "1").add("foo", "null").add("cnt", 8L).getRecord();
    List<BulletRecord> results = asList(expectedA, expectedB);
    Tuple expected = TupleUtils.makeTuple(TupleClassifier.Type.RESULT_TUPLE, "42", Clip.of(results).asJSON(), COMPLETED);
    Tuple tick = TupleUtils.makeTuple(TupleClassifier.Type.TICK_TUPLE);
    bolt.execute(tick);
    for (int i = 0; i < BulletStormConfig.DEFAULT_JOIN_BOLT_QUERY_TICK_TIMEOUT - 1; ++i) {
        bolt.execute(tick);
        Assert.assertFalse(wasResultEmittedTo(TopologyConstants.RESULT_STREAM, expected));
    }
    bolt.execute(tick);
    Assert.assertTrue(wasResultEmittedTo(TopologyConstants.RESULT_STREAM, expected));
    Tuple metadata = TupleUtils.makeTuple(TupleClassifier.Type.FEEDBACK_TUPLE, "42", new Metadata(Metadata.Signal.COMPLETE, null));
    Assert.assertTrue(wasMetadataEmittedTo(TopologyConstants.FEEDBACK_STREAM, metadata));
    Assert.assertEquals(collector.getAllEmittedTo(TopologyConstants.RESULT_STREAM).count(), 1);
    Assert.assertEquals(collector.getAllEmittedTo(TopologyConstants.FEEDBACK_STREAM).count(), 1);
}

Example 7 with COUNT

use of com.yahoo.bullet.aggregations.grouping.GroupOperation.GroupOperationType.COUNT in project bullet-storm by yahoo.

the class JoinBoltTest method testDistribution.

@Test
public void testDistribution() {
    BulletConfig bulletConfig = DistributionTest.makeConfiguration(10, 128);
    Distribution distribution = DistributionTest.makeDistribution(bulletConfig, makeAttributes(Distribution.Type.PMF, 3), "field", 10, null);
    IntStream.range(0, 50).mapToObj(i -> RecordBox.get().add("field", i).getRecord()).forEach(distribution::consume);
    byte[] first = distribution.getData();
    distribution = DistributionTest.makeDistribution(bulletConfig, makeAttributes(Distribution.Type.PMF, 3), "field", 10, null);
    IntStream.range(50, 101).mapToObj(i -> RecordBox.get().add("field", i).getRecord()).forEach(distribution::consume);
    byte[] second = distribution.getData();
    bolt = new DonableJoinBolt(config, 2, true);
    setup(bolt);
    Tuple query = TupleUtils.makeIDTuple(TupleClassifier.Type.QUERY_TUPLE, "42", makeAggregationQuery(DISTRIBUTION, 10, Distribution.Type.PMF, "field", null, null, null, null, 3), EMPTY);
    bolt.execute(query);
    sendRawByteTuplesTo(bolt, "42", asList(first, second));
    BulletRecord expectedA = RecordBox.get().add(RANGE_FIELD, NEGATIVE_INFINITY_START + SEPARATOR + 0.0 + END_EXCLUSIVE).add(COUNT_FIELD, 0.0).add(PROBABILITY_FIELD, 0.0).getRecord();
    BulletRecord expectedB = RecordBox.get().add(RANGE_FIELD, START_INCLUSIVE + 0.0 + SEPARATOR + 50.0 + END_EXCLUSIVE).add(COUNT_FIELD, 50.0).add(PROBABILITY_FIELD, 50.0 / 101).getRecord();
    BulletRecord expectedC = RecordBox.get().add(RANGE_FIELD, START_INCLUSIVE + 50.0 + SEPARATOR + 100.0 + END_EXCLUSIVE).add(COUNT_FIELD, 50.0).add(PROBABILITY_FIELD, 50.0 / 101).getRecord();
    BulletRecord expectedD = RecordBox.get().add(RANGE_FIELD, START_INCLUSIVE + 100.0 + SEPARATOR + POSITIVE_INFINITY_END).add(COUNT_FIELD, 1.0).add(PROBABILITY_FIELD, 1.0 / 101).getRecord();
    List<BulletRecord> results = asList(expectedA, expectedB, expectedC, expectedD);
    Tuple expected = TupleUtils.makeTuple(TupleClassifier.Type.RESULT_TUPLE, "42", Clip.of(results).asJSON(), COMPLETED);
    Tuple tick = TupleUtils.makeTuple(TupleClassifier.Type.TICK_TUPLE);
    bolt.execute(tick);
    for (int i = 0; i < BulletStormConfig.DEFAULT_JOIN_BOLT_QUERY_TICK_TIMEOUT - 1; ++i) {
        bolt.execute(tick);
        Assert.assertFalse(wasResultEmittedTo(TopologyConstants.RESULT_STREAM, expected));
    }
    bolt.execute(tick);
    Assert.assertTrue(wasResultEmittedTo(TopologyConstants.RESULT_STREAM, expected));
    Tuple metadata = TupleUtils.makeTuple(TupleClassifier.Type.FEEDBACK_TUPLE, "42", new Metadata(Metadata.Signal.COMPLETE, null));
    Assert.assertTrue(wasMetadataEmittedTo(TopologyConstants.FEEDBACK_STREAM, metadata));
    Assert.assertEquals(collector.getAllEmittedTo(TopologyConstants.RESULT_STREAM).count(), 1);
    Assert.assertEquals(collector.getAllEmittedTo(TopologyConstants.FEEDBACK_STREAM).count(), 1);
}

Example 8 with COUNT

use of com.yahoo.bullet.aggregations.grouping.GroupOperation.GroupOperationType.COUNT in project bullet-core by yahoo.

the class GroupByTest method testCombining.

@Test
public void testCombining() {
    List<String> fields = asList("fieldA", "fieldB");
    GroupBy groupBy = makeGroupBy(fields, 5, makeGroupOperation(COUNT, null, null), makeGroupOperation(SUM, "price", "priceSum"));
    BulletRecord recordA = RecordBox.get().add("fieldA", "foo").add("fieldB", "bar").add("price", 3).getRecord();
    BulletRecord recordB = RecordBox.get().add("fieldA", "null").add("fieldB", "bar").add("price", 1).getRecord();
    IntStream.range(0, 10).mapToObj(i -> recordA).forEach(groupBy::consume);
    IntStream.range(0, 9).mapToObj(i -> recordB).forEach(groupBy::consume);
    IntStream.range(0, 20).mapToObj(i -> recordA).forEach(groupBy::consume);
    groupBy.consume(recordB);
    byte[] firstSerialized = groupBy.getData();
    // Remake it
    groupBy = makeGroupBy(fields, 5, makeGroupOperation(COUNT, null, null), makeGroupOperation(SUM, "price", "priceSum"));
    BulletRecord recordC = RecordBox.get().add("fieldA", "foo").add("fieldB", "bar").add("price", 3).getRecord();
    BulletRecord recordD = RecordBox.get().addNull("fieldA").addNull("fieldB").add("price", 10).getRecord();
    IntStream.range(0, 30).mapToObj(i -> recordC).forEach(groupBy::consume);
    IntStream.range(0, 10).mapToObj(i -> recordD).forEach(groupBy::consume);
    byte[] secondSerialized = groupBy.getData();
    groupBy = makeGroupBy(fields, 5, makeGroupOperation(COUNT, null, null), makeGroupOperation(SUM, "price", "priceSum"));
    groupBy.combine(firstSerialized);
    groupBy.combine(secondSerialized);
    Clip aggregate = groupBy.getResult();
    Assert.assertNotNull(aggregate);
    List<BulletRecord> records = aggregate.getRecords();
    Assert.assertEquals(records.size(), 3);
    // count = 10 + 20 + 30, price = 10*3 + 20*3 + 30*3
    BulletRecord expectedA = RecordBox.get().add("fieldA", "foo").add("fieldB", "bar").add(COUNT.getName(), 60L).add("priceSum", 180.0).getRecord();
    // count = 9 + 1, price = 9*1 + 1*1
    BulletRecord expectedB = RecordBox.get().add("fieldA", "null").add("fieldB", "bar").add(COUNT.getName(), 10L).add("priceSum", 10.0).getRecord();
    // count = 10, price = 10*10
    BulletRecord expectedC = RecordBox.get().add("fieldA", "null").add("fieldB", "null").add(COUNT.getName(), 10L).add("priceSum", 100.0).getRecord();
    assertContains(records, expectedA);
    assertContains(records, expectedB);
    assertContains(records, expectedC);
    Assert.assertEquals(groupBy.getRecords(), aggregate.getRecords());
    Assert.assertEquals(groupBy.getMetadata().asMap(), aggregate.getMeta().asMap());
}

Example 9 with COUNT

use of com.yahoo.bullet.aggregations.grouping.GroupOperation.GroupOperationType.COUNT in project bullet-storm by yahoo.

the class JoinBoltTest method testGroupBy.

@Test
public void testGroupBy() {
    final int entries = 16;
    BulletConfig bulletConfig = GroupByTest.makeConfiguration(entries);
    GroupBy groupBy = GroupByTest.makeGroupBy(bulletConfig, singletonMap("fieldA", "A"), entries, AggregationUtils.makeGroupOperation(COUNT, null, "cnt"), AggregationUtils.makeGroupOperation(SUM, "fieldB", "sumB"));
    IntStream.range(0, 256).mapToObj(i -> RecordBox.get().add("fieldA", i % 16).add("fieldB", i / 16).getRecord()).forEach(groupBy::consume);
    byte[] first = groupBy.getData();
    groupBy = GroupByTest.makeGroupBy(bulletConfig, singletonMap("fieldA", "A"), entries, AggregationUtils.makeGroupOperation(COUNT, null, "cnt"), AggregationUtils.makeGroupOperation(SUM, "fieldB", "sumB"));
    IntStream.range(256, 1024).mapToObj(i -> RecordBox.get().add("fieldA", i % 16).add("fieldB", i / 16).getRecord()).forEach(groupBy::consume);
    byte[] second = groupBy.getData();
    // Send generated data to JoinBolt
    bolt = new DonableJoinBolt(config, 2, true);
    setup(bolt);
    List<GroupOperation> operations = asList(new GroupOperation(COUNT, null, "cnt"), new GroupOperation(SUM, "fieldB", "sumB"));
    String queryString = makeGroupFilterQuery("ts", singletonList("1"), EQUALS, GROUP, entries, operations, Pair.of("fieldA", "A"));
    Tuple query = TupleUtils.makeIDTuple(TupleClassifier.Type.QUERY_TUPLE, "42", queryString, EMPTY);
    bolt.execute(query);
    sendRawByteTuplesTo(bolt, "42", asList(first, second));
    Tuple tick = TupleUtils.makeTuple(TupleClassifier.Type.TICK_TUPLE);
    bolt.execute(tick);
    for (int i = 0; i < BulletStormConfig.DEFAULT_JOIN_BOLT_QUERY_TICK_TIMEOUT - 1; ++i) {
        bolt.execute(tick);
        Assert.assertEquals(collector.getEmittedCount(), 0);
    }
    bolt.execute(tick);
    Assert.assertEquals(collector.getEmittedCount(), 2);
    String response = (String) collector.getMthElementFromNthTupleEmittedTo(TopologyConstants.RESULT_STREAM, 1, 1).get();
    JsonParser parser = new JsonParser();
    JsonObject actual = parser.parse(response).getAsJsonObject();
    JsonArray actualRecords = actual.get(Clip.RECORDS_KEY).getAsJsonArray();
    Assert.assertEquals(actualRecords.size(), 16);
}