Examples with BulletConfig com.yahoo.bullet.common.BulletConfig used on opensource projects

Example 1 with BulletConfig

use of com.yahoo.bullet.common.BulletConfig in project bullet-core by yahoo.

the class GroupByTest method makeConfiguration.

public static BulletConfig makeConfiguration(int resizeFactor, float sampling, String separator, int k) {
    BulletConfig config = new BulletConfig();
    config.set(BulletConfig.GROUP_AGGREGATION_SKETCH_ENTRIES, k);
    config.set(BulletConfig.GROUP_AGGREGATION_SKETCH_RESIZE_FACTOR, resizeFactor);
    config.set(BulletConfig.GROUP_AGGREGATION_SKETCH_SAMPLING, sampling);
    config.set(BulletConfig.AGGREGATION_COMPOSITE_FIELD_SEPARATOR, separator);
    return config;
}

Example 2 with BulletConfig

use of com.yahoo.bullet.common.BulletConfig in project bullet-core by yahoo.

the class TopKTest method makeConfiguration.

public static BulletConfig makeConfiguration(ErrorType errorType, int maxMapSize) {
    BulletConfig config = new BulletConfig();
    config.set(BulletConfig.TOP_K_AGGREGATION_SKETCH_ENTRIES, maxMapSize);
    config.set(BulletConfig.TOP_K_AGGREGATION_SKETCH_ERROR_TYPE, errorType == ErrorType.NO_FALSE_POSITIVES ? TopK.NO_FALSE_POSITIVES : TopK.NO_FALSE_NEGATIVES);
    return config;
}

Example 3 with BulletConfig

use of com.yahoo.bullet.common.BulletConfig in project bullet-core by yahoo.

the class CountDistinctTest method testMultipleFieldsCountDistinct.

@Test
public void testMultipleFieldsCountDistinct() {
    BulletConfig config = makeConfiguration(4, 512);
    CountDistinct countDistinct = makeCountDistinct(config, makeAttributes("myCount"), asList("fieldA", "fieldB"));
    IntStream.range(0, 256).mapToObj(i -> RecordBox.get().add("fieldA", i).add("fieldB", 255 - i).getRecord()).forEach(countDistinct::consume);
    IntStream.range(0, 256).mapToObj(i -> RecordBox.get().add("fieldA", i).add("fieldB", 255 - i).getRecord()).forEach(countDistinct::consume);
    Clip clip = countDistinct.getResult();
    Assert.assertEquals(clip.getRecords().size(), 1);
    BulletRecord actual = clip.getRecords().get(0);
    BulletRecord expected = RecordBox.get().add("myCount", 256.0).getRecord();
    Assert.assertEquals(actual, expected);
    Assert.assertEquals(countDistinct.getRecords(), clip.getRecords());
    Assert.assertEquals(countDistinct.getMetadata().asMap(), countDistinct.getMetadata().asMap());
}

Example 4 with BulletConfig

use of com.yahoo.bullet.common.BulletConfig in project bullet-core by yahoo.

the class CountDistinctTest method testMultipleFieldsCountDistinctAmbiguity.

@Test
public void testMultipleFieldsCountDistinctAmbiguity() {
    BulletConfig config = makeConfiguration(4, 512);
    String s = BulletConfig.DEFAULT_AGGREGATION_COMPOSITE_FIELD_SEPARATOR;
    CountDistinct countDistinct = makeCountDistinct(config, makeAttributes("myCount"), asList("fieldA", "fieldB"));
    BulletRecord first = RecordBox.get().add("fieldA", s).add("fieldB", s + s).getRecord();
    BulletRecord second = RecordBox.get().add("fieldA", s + s).add("fieldB", s).getRecord();
    // first and second will look the same to the Sketch. third will not
    BulletRecord third = RecordBox.get().add("fieldA", s + s).add("fieldB", s + s).getRecord();
    countDistinct.consume(first);
    countDistinct.consume(second);
    countDistinct.consume(third);
    Clip clip = countDistinct.getResult();
    Assert.assertEquals(clip.getRecords().size(), 1);
    BulletRecord actual = clip.getRecords().get(0);
    BulletRecord expected = RecordBox.get().add("myCount", 2.0).getRecord();
    Assert.assertEquals(actual, expected);
    Assert.assertEquals(countDistinct.getRecords(), clip.getRecords());
    Assert.assertEquals(countDistinct.getMetadata().asMap(), countDistinct.getMetadata().asMap());
}

Example 5 with BulletConfig

use of com.yahoo.bullet.common.BulletConfig in project bullet-core by yahoo.

the class CountDistinctTest method testCombiningExact.

@Test
public void testCombiningExact() {
    BulletConfig config = makeConfiguration(4, 1024);
    CountDistinct countDistinct = makeCountDistinct(config, makeAttributes("myCount"), asList("field"));
    IntStream.range(0, 512).mapToObj(i -> RecordBox.get().add("field", i).getRecord()).forEach(countDistinct::consume);
    byte[] firstAggregate = countDistinct.getData();
    // Another one
    countDistinct = makeCountDistinct(config, makeAttributes("myCount"), asList("field"));
    IntStream.range(256, 768).mapToObj(i -> RecordBox.get().add("field", i).getRecord()).forEach(countDistinct::consume);
    byte[] secondAggregate = countDistinct.getData();
    // Final one
    countDistinct = makeCountDistinct(config, makeAttributes("myCount"), asList("field"), Pair.of(Concept.SKETCH_METADATA, "stats"), Pair.of(Concept.SKETCH_ESTIMATED_RESULT, "est"));
    countDistinct.combine(firstAggregate);
    countDistinct.combine(secondAggregate);
    Clip clip = countDistinct.getResult();
    Map<String, Object> meta = clip.getMeta().asMap();
    Assert.assertEquals(meta.size(), 1);
    Assert.assertTrue(meta.containsKey("stats"));
    Map<String, Object> stats = (Map<String, Object>) meta.get("stats");
    Assert.assertEquals(stats.size(), 1);
    Assert.assertFalse((Boolean) stats.get("est"));
    Assert.assertEquals(clip.getRecords().size(), 1);
    BulletRecord actual = clip.getRecords().get(0);
    BulletRecord expected = RecordBox.get().add("myCount", 768.0).getRecord();
    Assert.assertEquals(actual, expected);
    Assert.assertEquals(countDistinct.getRecords(), clip.getRecords());
    Assert.assertEquals(countDistinct.getMetadata().asMap(), countDistinct.getMetadata().asMap());
}