Examples with GroupOperation com.yahoo.bullet.aggregations.grouping.GroupOperation used on opensource projects

Example 1 with GroupOperation

use of com.yahoo.bullet.aggregations.grouping.GroupOperation in project bullet-storm by yahoo.

the class FilterBoltTest method testGroupAllCount.

@Test
public void testGroupAllCount() {
    // 15 Records will be consumed
    bolt = ComponentUtils.prepare(new DonableFilterBolt(15, new BulletStormConfig()), collector);
    Tuple query = makeIDTuple(TupleClassifier.Type.QUERY_TUPLE, "42", makeGroupFilterQuery("timestamp", asList("1", "2"), EQUALS, GROUP, 1, singletonList(new GroupOperation(COUNT, null, "cnt"))), METADATA);
    bolt.execute(query);
    BulletRecord record = RecordBox.get().add("timestamp", "1").getRecord();
    Tuple matching = makeRecordTuple(record);
    IntStream.range(0, 10).forEach(i -> bolt.execute(matching));
    BulletRecord another = RecordBox.get().getRecord();
    Tuple nonMatching = makeRecordTuple(another);
    IntStream.range(0, 5).forEach(i -> bolt.execute(nonMatching));
    bolt.execute(nonMatching);
    // Two to flush bolt
    Tuple tick = TupleUtils.makeTuple(TupleClassifier.Type.TICK_TUPLE);
    bolt.execute(tick);
    bolt.execute(tick);
    Assert.assertEquals(collector.getEmittedCount(), 1);
    GroupData actual = SerializerDeserializer.fromBytes(getRawPayloadOfNthTuple(1));
    BulletRecord expected = RecordBox.get().add("cnt", 10).getRecord();
    Assert.assertTrue(isEqual(actual, expected));
}

Example 2 with GroupOperation

use of com.yahoo.bullet.aggregations.grouping.GroupOperation in project bullet-storm by yahoo.

the class JoinBoltTest method testQueryCountingMetrics.

@Test
public void testQueryCountingMetrics() {
    config.set(BulletStormConfig.TOPOLOGY_METRICS_BUILT_IN_ENABLE, true);
    config.validate();
    bolt = new DonableJoinBolt(config, 2, true);
    setup(bolt);
    String filterQuery = makeGroupFilterQuery("timestamp", asList("1", "2"), EQUALS, GROUP, 1, singletonList(new GroupOperation(COUNT, null, "cnt")));
    Tuple query = TupleUtils.makeIDTuple(TupleClassifier.Type.QUERY_TUPLE, "42", filterQuery, EMPTY);
    Assert.assertEquals(context.getLongMetric(TopologyConstants.CREATED_QUERIES_METRIC), Long.valueOf(0));
    Assert.assertEquals(context.getLongMetric(TopologyConstants.ACTIVE_QUERIES_METRIC), Long.valueOf(0));
    Assert.assertEquals(context.getLongMetric(TopologyConstants.IMPROPER_QUERIES_METRIC), Long.valueOf(0));
    bolt.execute(query);
    Assert.assertEquals(context.getLongMetric(TopologyConstants.CREATED_QUERIES_METRIC), Long.valueOf(1));
    Assert.assertEquals(context.getLongMetric(TopologyConstants.ACTIVE_QUERIES_METRIC), Long.valueOf(1));
    sendRawByteTuplesTo(bolt, "42", singletonList(getGroupDataWithCount("cnt", 21)));
    sendRawByteTuplesTo(bolt, "42", singletonList(getGroupDataWithCount("cnt", 21)));
    List<BulletRecord> result = singletonList(RecordBox.get().add("cnt", 42L).getRecord());
    Tuple expected = TupleUtils.makeTuple(TupleClassifier.Type.RESULT_TUPLE, "42", Clip.of(result).asJSON(), COMPLETED);
    Tuple tick = TupleUtils.makeTuple(TupleClassifier.Type.TICK_TUPLE);
    // Should cause an expiry and starts buffering the query for the query tickout
    bolt.execute(tick);
    // We need to tick the default query tickout to make the query emit
    for (int i = 0; i < BulletStormConfig.DEFAULT_JOIN_BOLT_QUERY_TICK_TIMEOUT - 1; ++i) {
        bolt.execute(tick);
    }
    Assert.assertFalse(wasResultEmittedTo(TopologyConstants.RESULT_STREAM, expected));
    Assert.assertEquals(context.getLongMetric(TopologyConstants.ACTIVE_QUERIES_METRIC), Long.valueOf(1));
    bolt.execute(tick);
    Assert.assertEquals(context.getLongMetric(TopologyConstants.ACTIVE_QUERIES_METRIC), Long.valueOf(0));
    Assert.assertTrue(wasResultEmittedTo(TopologyConstants.RESULT_STREAM, expected));
    Tuple metadata = TupleUtils.makeTuple(TupleClassifier.Type.FEEDBACK_TUPLE, "42", new Metadata(Metadata.Signal.COMPLETE, null));
    Assert.assertTrue(wasMetadataEmittedTo(TopologyConstants.FEEDBACK_STREAM, metadata));
    Assert.assertEquals(collector.getAllEmittedTo(TopologyConstants.RESULT_STREAM).count(), 1);
    Assert.assertEquals(collector.getAllEmittedTo(TopologyConstants.FEEDBACK_STREAM).count(), 1);
    bolt.execute(query);
    Assert.assertEquals(context.getLongMetric(TopologyConstants.CREATED_QUERIES_METRIC), Long.valueOf(2));
    Assert.assertEquals(context.getLongMetric(TopologyConstants.ACTIVE_QUERIES_METRIC), Long.valueOf(1));
    Assert.assertEquals(context.getLongMetric(TopologyConstants.IMPROPER_QUERIES_METRIC), Long.valueOf(0));
}

Example 3 with GroupOperation

use of com.yahoo.bullet.aggregations.grouping.GroupOperation in project bullet-storm by yahoo.

the class JoinBoltTest method testCustomMetricEmitInterval.

@Test
public void testCustomMetricEmitInterval() {
    Map<String, Number> mapping = new HashMap<>();
    mapping.put(TopologyConstants.ACTIVE_QUERIES_METRIC, 1);
    mapping.put(BulletStormConfig.DEFAULT_BUILT_IN_METRICS_INTERVAL_KEY, 10);
    config.set(BulletStormConfig.TOPOLOGY_METRICS_BUILT_IN_EMIT_INTERVAL_MAPPING, mapping);
    config.set(BulletStormConfig.TOPOLOGY_METRICS_BUILT_IN_ENABLE, true);
    bolt = new DonableJoinBolt(config, BulletStormConfig.DEFAULT_JOIN_BOLT_QUERY_TICK_TIMEOUT, false);
    setup(bolt);
    String filterQuery = makeGroupFilterQuery("timestamp", asList("1", "2"), EQUALS, GROUP, 1, singletonList(new GroupOperation(COUNT, null, "cnt")));
    Tuple query = TupleUtils.makeIDTuple(TupleClassifier.Type.QUERY_TUPLE, "42", filterQuery, EMPTY);
    Assert.assertEquals(context.getLongMetric(10, TopologyConstants.CREATED_QUERIES_METRIC), Long.valueOf(0));
    Assert.assertEquals(context.getLongMetric(1, TopologyConstants.ACTIVE_QUERIES_METRIC), Long.valueOf(0));
    Assert.assertEquals(context.getLongMetric(10, TopologyConstants.IMPROPER_QUERIES_METRIC), Long.valueOf(0));
    bolt.execute(query);
    Assert.assertEquals(context.getLongMetric(10, TopologyConstants.CREATED_QUERIES_METRIC), Long.valueOf(1));
    Assert.assertEquals(context.getLongMetric(1, TopologyConstants.ACTIVE_QUERIES_METRIC), Long.valueOf(1));
    Tuple tick = TupleUtils.makeTuple(TupleClassifier.Type.TICK_TUPLE);
    for (int i = 0; i <= BulletStormConfig.DEFAULT_JOIN_BOLT_QUERY_TICK_TIMEOUT; ++i) {
        bolt.execute(tick);
    }
    List<BulletRecord> result = singletonList(RecordBox.get().add("cnt", 0L).getRecord());
    Tuple expected = TupleUtils.makeTuple(TupleClassifier.Type.RESULT_TUPLE, "42", Clip.of(result).asJSON(), COMPLETED);
    Assert.assertTrue(wasResultEmittedTo(TopologyConstants.RESULT_STREAM, expected));
    Tuple metadata = TupleUtils.makeTuple(TupleClassifier.Type.FEEDBACK_TUPLE, "42", new Metadata(Metadata.Signal.COMPLETE, null));
    Assert.assertTrue(wasMetadataEmittedTo(TopologyConstants.FEEDBACK_STREAM, metadata));
    Assert.assertEquals(context.getLongMetric(10, TopologyConstants.CREATED_QUERIES_METRIC), Long.valueOf(1));
    Assert.assertEquals(context.getLongMetric(1, TopologyConstants.ACTIVE_QUERIES_METRIC), Long.valueOf(0));
    Assert.assertEquals(context.getLongMetric(10, TopologyConstants.IMPROPER_QUERIES_METRIC), Long.valueOf(0));
}

Example 4 with GroupOperation

use of com.yahoo.bullet.aggregations.grouping.GroupOperation in project bullet-storm by yahoo.

the class JoinBoltTest method testCounting.

@Test
public void testCounting() {
    bolt = new DonableJoinBolt(config, 5, true);
    setup(bolt);
    String filterQuery = makeGroupFilterQuery("timestamp", asList("1", "2"), EQUALS, GROUP, 1, singletonList(new GroupOperation(COUNT, null, "cnt")));
    Tuple query = TupleUtils.makeIDTuple(TupleClassifier.Type.QUERY_TUPLE, "42", filterQuery, EMPTY);
    bolt.execute(query);
    // Send 5 GroupData with counts 1, 2, 3, 4, 5 to the JoinBolt
    IntStream.range(1, 6).forEach(i -> sendRawByteTuplesTo(bolt, "42", singletonList(getGroupDataWithCount("cnt", i))));
    // 1 + 2 + 3 + 4 + 5
    List<BulletRecord> result = singletonList(RecordBox.get().add("cnt", 15L).getRecord());
    Tuple expected = TupleUtils.makeTuple(TupleClassifier.Type.RESULT_TUPLE, "42", Clip.of(result).asJSON(), COMPLETED);
    Tuple tick = TupleUtils.makeTuple(TupleClassifier.Type.TICK_TUPLE);
    // Should starts buffering the query for the query tickout
    bolt.execute(tick);
    for (int i = 0; i < BulletStormConfig.DEFAULT_JOIN_BOLT_QUERY_TICK_TIMEOUT - 1; ++i) {
        bolt.execute(tick);
        Assert.assertFalse(wasResultEmittedTo(TopologyConstants.RESULT_STREAM, expected));
    }
    bolt.execute(tick);
    Assert.assertTrue(wasResultEmittedTo(TopologyConstants.RESULT_STREAM, expected));
    Tuple metadata = TupleUtils.makeTuple(TupleClassifier.Type.FEEDBACK_TUPLE, "42", new Metadata(Metadata.Signal.COMPLETE, null));
    Assert.assertTrue(wasMetadataEmittedTo(TopologyConstants.FEEDBACK_STREAM, metadata));
    Assert.assertEquals(collector.getAllEmittedTo(TopologyConstants.RESULT_STREAM).count(), 1);
    Assert.assertEquals(collector.getAllEmittedTo(TopologyConstants.FEEDBACK_STREAM).count(), 1);
}

Example 5 with GroupOperation

use of com.yahoo.bullet.aggregations.grouping.GroupOperation in project bullet-core by yahoo.

the class GroupAllTest method testCombiningMetrics.

@Test
public void testCombiningMetrics() {
    List<GroupOperation> operations = asList(new GroupOperation(GroupOperation.GroupOperationType.COUNT, null, "myCount"), new GroupOperation(GroupOperation.GroupOperationType.MIN, "minField", "myMin"), new GroupOperation(GroupOperation.GroupOperationType.AVG, "groupField", "groupAvg"), new GroupOperation(GroupOperation.GroupOperationType.MIN, "groupField", "groupMin"), new GroupOperation(GroupOperation.GroupOperationType.SUM, "groupField", "groupSum"));
    GroupAll groupAll = makeGroupAll(operations);
    groupAll.consume(RecordBox.get().add("minField", -8.8).add("groupField", 3.14).getRecord());
    groupAll.consume(RecordBox.get().add("minField", 0.0).addNull("groupField").getRecord());
    groupAll.consume(RecordBox.get().add("minField", 51.44).add("groupField", -4.88).getRecord());
    GroupAll another = makeGroupAll(operations);
    another.consume(RecordBox.get().add("minField", -8.8).add("groupField", 12345.67).getRecord());
    another.consume(RecordBox.get().addNull("minField").add("groupField", 2.718).getRecord());
    another.consume(RecordBox.get().add("minField", -51.0).addNull("groupField").getRecord());
    another.consume(RecordBox.get().add("minField", 0).add("groupField", 1).getRecord());
    another.consume(RecordBox.get().add("minField", 44.8).add("groupField", -51.44).getRecord());
    byte[] serialized = another.getData();
    groupAll.combine(serialized);
    Assert.assertNotNull(groupAll.getData());
    List<BulletRecord> aggregate = groupAll.getResult().getRecords();
    Assert.assertEquals(aggregate.size(), 1);
    BulletRecord actual = aggregate.get(0);
    BulletRecord expected = RecordBox.get().add("myCount", 8L).add("myMin", -51.0).add("groupAvg", 2049.368).add("groupMin", -51.44).add("groupSum", 12296.208).getRecord();
    Assert.assertTrue(actual.equals(expected));
    Assert.assertEquals(groupAll.getRecords(), aggregate);
    Assert.assertEquals(groupAll.getMetadata().asMap(), groupAll.getResult().getMeta().asMap());
}