Examples with CountDistinct com.yahoo.bullet.aggregations.CountDistinct used on opensource projects

Example 1 with CountDistinct

use of com.yahoo.bullet.aggregations.CountDistinct in project bullet-storm by yahoo.

the class JoinBoltTest method testCountDistinct.

@Test
public void testCountDistinct() {
    BulletConfig bulletConfig = CountDistinctTest.makeConfiguration(8, 512);
    CountDistinct distinct = CountDistinctTest.makeCountDistinct(bulletConfig, singletonList("field"));
    IntStream.range(0, 256).mapToObj(i -> RecordBox.get().add("field", i).getRecord()).forEach(distinct::consume);
    byte[] first = distinct.getData();
    distinct = CountDistinctTest.makeCountDistinct(bulletConfig, singletonList("field"));
    IntStream.range(128, 256).mapToObj(i -> RecordBox.get().add("field", i).getRecord()).forEach(distinct::consume);
    byte[] second = distinct.getData();
    // Send generated data to JoinBolt
    bolt = new DonableJoinBolt(config, 2, true);
    setup(bolt);
    Tuple query = TupleUtils.makeIDTuple(TupleClassifier.Type.QUERY_TUPLE, "42", makeAggregationQuery(COUNT_DISTINCT, 1, null, Pair.of("field", "field")), EMPTY);
    bolt.execute(query);
    sendRawByteTuplesTo(bolt, "42", asList(first, second));
    List<BulletRecord> result = singletonList(RecordBox.get().add(CountDistinct.DEFAULT_NEW_NAME, 256.0).getRecord());
    Tuple expected = TupleUtils.makeTuple(TupleClassifier.Type.RESULT_TUPLE, "42", Clip.of(result).asJSON(), COMPLETED);
    Tuple tick = TupleUtils.makeTuple(TupleClassifier.Type.TICK_TUPLE);
    bolt.execute(tick);
    for (int i = 0; i < BulletStormConfig.DEFAULT_JOIN_BOLT_QUERY_TICK_TIMEOUT - 1; ++i) {
        bolt.execute(tick);
        Assert.assertFalse(wasResultEmittedTo(TopologyConstants.RESULT_STREAM, expected));
    }
    bolt.execute(tick);
    Assert.assertTrue(wasResultEmittedTo(TopologyConstants.RESULT_STREAM, expected));
    Tuple metadata = TupleUtils.makeTuple(TupleClassifier.Type.FEEDBACK_TUPLE, "42", new Metadata(Metadata.Signal.COMPLETE, null));
    Assert.assertTrue(wasMetadataEmittedTo(TopologyConstants.FEEDBACK_STREAM, metadata));
    Assert.assertEquals(collector.getAllEmittedTo(TopologyConstants.RESULT_STREAM).count(), 1);
    Assert.assertEquals(collector.getAllEmittedTo(TopologyConstants.FEEDBACK_STREAM).count(), 1);
}

Example 2 with CountDistinct

use of com.yahoo.bullet.aggregations.CountDistinct in project bullet-storm by yahoo.

the class FilterBoltTest method testCountDistinct.

@Test
public void testCountDistinct() {
    // 256 Records will be consumed
    BulletStormConfig config = new BulletStormConfig(CountDistinctTest.makeConfiguration(8, 512));
    bolt = ComponentUtils.prepare(new DonableFilterBolt(256, config), collector);
    Tuple query = makeIDTuple(TupleClassifier.Type.QUERY_TUPLE, "42", makeAggregationQuery(COUNT_DISTINCT, 1, null, Pair.of("field", "field")), METADATA);
    bolt.execute(query);
    IntStream.range(0, 256).mapToObj(i -> RecordBox.get().add("field", i).getRecord()).map(FilterBoltTest::makeRecordTuple).forEach(bolt::execute);
    Assert.assertEquals(collector.getEmittedCount(), 0);
    Tuple tick = TupleUtils.makeTuple(TupleClassifier.Type.TICK_TUPLE);
    bolt.execute(tick);
    bolt.execute(tick);
    Assert.assertEquals(collector.getEmittedCount(), 1);
    byte[] rawData = getRawPayloadOfNthTuple(1);
    Assert.assertNotNull(rawData);
    CountDistinct distinct = CountDistinctTest.makeCountDistinct(config, singletonList("field"));
    distinct.combine(rawData);
    BulletRecord actual = distinct.getRecords().get(0);
    BulletRecord expected = RecordBox.get().add(CountDistinct.DEFAULT_NEW_NAME, 256.0).getRecord();
    Assert.assertEquals(actual, expected);
}