Examples with CountDistinct com.yahoo.bullet.query.aggregations.CountDistinct used on opensource projects

Example 1 with CountDistinct

use of com.yahoo.bullet.query.aggregations.CountDistinct in project bullet-core by yahoo.

the class ThetaSketchingStrategyTest method testSingleFieldExactCountDistinctWithoutDuplicates.

@Test
public void testSingleFieldExactCountDistinctWithoutDuplicates() {
    ThetaSketchingStrategy countDistinct = makeCountDistinct(Collections.singletonList("field"), DEFAULT_NAME);
    IntStream.range(0, 1000).mapToObj(i -> RecordBox.get().add("field", i).getRecord()).forEach(countDistinct::consume);
    Assert.assertNotNull(countDistinct.getData());
    List<BulletRecord> aggregate = countDistinct.getResult().getRecords();
    Assert.assertEquals(aggregate.size(), 1);
    BulletRecord actual = aggregate.get(0);
    BulletRecord expected = RecordBox.get().add(DEFAULT_NAME, 1000L).getRecord();
    Assert.assertEquals(actual, expected);
    Assert.assertEquals(countDistinct.getRecords(), aggregate);
    Assert.assertEquals(countDistinct.getMetadata().asMap(), countDistinct.getMetadata().asMap());
}

Example 2 with CountDistinct

use of com.yahoo.bullet.query.aggregations.CountDistinct in project bullet-core by yahoo.

the class ThetaSketchingStrategyTest method testCombiningAndConsuming.

@Test
public void testCombiningAndConsuming() {
    BulletConfig config = makeConfiguration(4, 1024);
    ThetaSketchingStrategy countDistinct = makeCountDistinct(config, Collections.singletonList("field"), "myCount");
    IntStream.range(0, 256).mapToObj(i -> RecordBox.get().add("field", i).getRecord()).forEach(countDistinct::consume);
    byte[] aggregate = countDistinct.getData();
    // New one
    countDistinct = makeCountDistinct(config, Collections.singletonList("field"), "myCount");
    IntStream.range(0, 768).mapToObj(i -> RecordBox.get().add("field", i).getRecord()).forEach(countDistinct::consume);
    countDistinct.combine(aggregate);
    Clip clip = countDistinct.getResult();
    Map<String, Object> meta = clip.getMeta().asMap();
    Assert.assertEquals(meta.size(), 0);
    Assert.assertEquals(clip.getRecords().size(), 1);
    BulletRecord actual = clip.getRecords().get(0);
    BulletRecord expected = RecordBox.get().add("myCount", 768L).getRecord();
    Assert.assertEquals(actual, expected);
    Assert.assertEquals(countDistinct.getRecords(), clip.getRecords());
    Assert.assertEquals(countDistinct.getMetadata().asMap(), countDistinct.getMetadata().asMap());
}

Example 3 with CountDistinct

use of com.yahoo.bullet.query.aggregations.CountDistinct in project bullet-core by yahoo.

the class ThetaSketchingStrategyTest method makeCountDistinct.

@SafeVarargs
public static ThetaSketchingStrategy makeCountDistinct(List<String> fields, String name, Map.Entry<Concept, String>... metadata) {
    CountDistinct aggregation = new CountDistinct(fields, name);
    BulletConfig configuration = makeConfiguration(8, 1024);
    return new ThetaSketchingStrategy(aggregation, addMetadata(configuration, metadata));
}

Example 4 with CountDistinct

use of com.yahoo.bullet.query.aggregations.CountDistinct in project bullet-core by yahoo.

the class ThetaSketchingStrategyTest method testSingleFieldExactCountDistinctWithDuplicates.

@Test
public void testSingleFieldExactCountDistinctWithDuplicates() {
    ThetaSketchingStrategy countDistinct = makeCountDistinct(Collections.singletonList("field"), DEFAULT_NAME);
    IntStream.range(0, 1000).mapToObj(i -> RecordBox.get().add("field", i).getRecord()).forEach(countDistinct::consume);
    IntStream.range(0, 1000).mapToObj(i -> RecordBox.get().add("field", i).getRecord()).forEach(countDistinct::consume);
    Assert.assertNotNull(countDistinct.getData());
    List<BulletRecord> aggregate = countDistinct.getResult().getRecords();
    Assert.assertEquals(aggregate.size(), 1);
    BulletRecord actual = aggregate.get(0);
    BulletRecord expected = RecordBox.get().add(DEFAULT_NAME, 1000L).getRecord();
    Assert.assertEquals(actual, expected);
    Assert.assertEquals(countDistinct.getRecords(), aggregate);
    Assert.assertEquals(countDistinct.getMetadata().asMap(), countDistinct.getMetadata().asMap());
}

Example 5 with CountDistinct

use of com.yahoo.bullet.query.aggregations.CountDistinct in project bullet-core by yahoo.

the class ThetaSketchingStrategyTest method testCombiningExact.

@Test
public void testCombiningExact() {
    BulletConfig config = makeConfiguration(4, 1024);
    ThetaSketchingStrategy countDistinct = makeCountDistinct(config, Collections.singletonList("field"), "myCount");
    IntStream.range(0, 512).mapToObj(i -> RecordBox.get().add("field", i).getRecord()).forEach(countDistinct::consume);
    byte[] firstAggregate = countDistinct.getData();
    // Another one
    countDistinct = makeCountDistinct(config, Collections.singletonList("field"), "myCount");
    IntStream.range(256, 768).mapToObj(i -> RecordBox.get().add("field", i).getRecord()).forEach(countDistinct::consume);
    byte[] secondAggregate = countDistinct.getData();
    // Final one
    countDistinct = makeCountDistinct(config, Collections.singletonList("field"), "myCount", Pair.of(Concept.SKETCH_METADATA, "stats"), Pair.of(Concept.SKETCH_ESTIMATED_RESULT, "est"));
    countDistinct.combine(firstAggregate);
    countDistinct.combine(secondAggregate);
    Clip clip = countDistinct.getResult();
    Map<String, Object> meta = clip.getMeta().asMap();
    Assert.assertEquals(meta.size(), 1);
    Assert.assertTrue(meta.containsKey("stats"));
    Map<String, Object> stats = (Map<String, Object>) meta.get("stats");
    Assert.assertEquals(stats.size(), 1);
    Assert.assertFalse((Boolean) stats.get("est"));
    Assert.assertEquals(clip.getRecords().size(), 1);
    BulletRecord actual = clip.getRecords().get(0);
    BulletRecord expected = RecordBox.get().add("myCount", 768L).getRecord();
    Assert.assertEquals(actual, expected);
    Assert.assertEquals(countDistinct.getRecords(), clip.getRecords());
    Assert.assertEquals(countDistinct.getMetadata().asMap(), countDistinct.getMetadata().asMap());
}