Examples with Clip com.yahoo.bullet.result.Clip used on opensource projects

Example 51 with Clip

use of com.yahoo.bullet.result.Clip in project bullet-core by yahoo.

the class TopKTest method testExactTopK.

@Test
public void testExactTopK() {
    TopK topK = makeTopK(asList("A", "B"), 64, 20);
    IntStream.range(0, 996).mapToObj(i -> RecordBox.get().add("A", String.valueOf(i % 3)).add("B", i % 4).getRecord()).forEach(topK::consume);
    Clip result = topK.getResult();
    Map<String, Object> metadata = (Map<String, Object>) result.getMeta().asMap().get("meta");
    Assert.assertEquals(metadata.size(), 5);
    Assert.assertFalse((Boolean) metadata.get("isEst"));
    Assert.assertEquals((String) metadata.get("family"), Family.FREQUENCY.getFamilyName());
    Assert.assertEquals(metadata.get("error"), 0L);
    Assert.assertEquals(metadata.get("n"), 996L);
    Assert.assertEquals(metadata.get("actives"), 12);
    List<BulletRecord> records = result.getRecords();
    Assert.assertEquals(records.size(), 12);
    Set<Integer> fields = new HashSet<>();
    for (BulletRecord actual : records) {
        Assert.assertEquals(actual.fieldCount(), 3);
        int fieldA = Integer.valueOf(actual.get("A").toString());
        int fieldB = Integer.valueOf(actual.get("B").toString());
        Assert.assertTrue(fieldA < 3);
        Assert.assertTrue(fieldB < 4);
        fields.add(fieldA * 3 + fieldB * 4);
        Assert.assertEquals(actual.get(TopK.DEFAULT_NEW_NAME), 83L);
    }
    Assert.assertEquals(fields.size(), 12);
    Assert.assertEquals(topK.getRecords(), records);
    Assert.assertEquals(topK.getMetadata().asMap(), result.getMeta().asMap());
}

Example 52 with Clip

use of com.yahoo.bullet.result.Clip in project bullet-core by yahoo.

the class QuerierTest method testExceptionWrapping.

@Test
public void testExceptionWrapping() {
    FailingScheme failingScheme = new FailingScheme(null, null, new BulletConfig());
    Querier querier = make(Querier.Mode.ALL, new Query());
    querier.setWindow(failingScheme);
    querier.consume(RecordBox.get().getRecord());
    querier.combine(new byte[0]);
    Assert.assertNull(querier.getData());
    Assert.assertNull(querier.getRecords());
    Meta meta = querier.getMetadata();
    Map<String, Object> actualMeta = meta.asMap();
    Assert.assertNotNull(actualMeta.get(Meta.ERROR_KEY));
    BulletError expected = BulletError.makeError("Getting metadata failure", Querier.TRY_AGAIN_LATER);
    Assert.assertEquals(actualMeta.get(Meta.ERROR_KEY), singletonList(expected));
    Clip actual = querier.getResult();
    Assert.assertNotNull(actual.getMeta());
    Assert.assertEquals(actual.getRecords().size(), 0);
    actualMeta = actual.getMeta().asMap();
    Assert.assertEquals(actualMeta.size(), 1);
    Assert.assertNotNull(actualMeta.get(Meta.ERROR_KEY));
    expected = BulletError.makeError("Getting result failure", Querier.TRY_AGAIN_LATER);
    Assert.assertEquals(actualMeta.get(Meta.ERROR_KEY), singletonList(expected));
    Assert.assertEquals(failingScheme.consumptionFailure, 1);
    Assert.assertEquals(failingScheme.combiningFailure, 1);
    Assert.assertEquals(failingScheme.serializingFailure, 1);
    Assert.assertEquals(failingScheme.aggregationFailure, 3);
}

Example 53 with Clip

use of com.yahoo.bullet.result.Clip in project bullet-core by yahoo.

the class BasicTest method testAllDataMethodsProxyToStrategy.

@Test
public void testAllDataMethodsProxyToStrategy() {
    addMetadata(config, ALL_METADATA);
    Basic basic = new Basic(strategy, null, config);
    Assert.assertEquals(strategy.getConsumeCalls(), 0);
    Assert.assertEquals(strategy.getCombineCalls(), 0);
    Assert.assertEquals(strategy.getDataCalls(), 0);
    Assert.assertEquals(strategy.getMetadataCalls(), 0);
    Assert.assertEquals(strategy.getRecordCalls(), 0);
    Assert.assertEquals(strategy.getResetCalls(), 0);
    basic.consume(null);
    Assert.assertEquals(strategy.getConsumeCalls(), 1);
    basic.combine(null);
    Assert.assertEquals(strategy.getCombineCalls(), 1);
    Assert.assertNull(basic.getData());
    Assert.assertEquals(strategy.getDataCalls(), 1);
    long timeNow = System.currentTimeMillis();
    Meta meta = basic.getMetadata();
    Assert.assertNotNull(meta);
    Map<String, Object> asMap = (Map<String, Object>) meta.asMap().get("window_stats");
    Assert.assertEquals(asMap.get("name"), Basic.NAME);
    Assert.assertEquals(asMap.get("num"), 1L);
    Assert.assertTrue(((Long) asMap.get("closed")) >= timeNow);
    Assert.assertEquals(strategy.getMetadataCalls(), 1);
    Assert.assertNull(basic.getRecords());
    Assert.assertEquals(strategy.getRecordCalls(), 1);
    Clip clip = basic.getResult();
    Map<String, Object> expected = (Map<String, Object>) meta.asMap().get("window_stats");
    Map<String, Object> actual = (Map<String, Object>) clip.getMeta().asMap().get("window_stats");
    Assert.assertEquals(actual.get("name"), expected.get("name"));
    Assert.assertEquals(actual.get("num"), expected.get("num"));
    Assert.assertTrue(((Long) actual.get("closed")) >= timeNow);
    Assert.assertEquals(clip.getRecords(), Collections.emptyList());
    // We will not call getResult because we use getMetadata and getRecords
    Assert.assertEquals(strategy.getResultCalls(), 0);
    basic.reset();
    Assert.assertEquals(strategy.getResetCalls(), 1);
    basic.resetForPartition();
    Assert.assertEquals(strategy.getResetCalls(), 2);
}

Example 54 with Clip

use of com.yahoo.bullet.result.Clip in project bullet-core by yahoo.

the class CountDistinctTest method testNewNamingOfResult.

@Test
public void testNewNamingOfResult() {
    BulletConfig config = makeConfiguration(4, 1024);
    CountDistinct countDistinct = makeCountDistinct(config, makeAttributes("myCount"), asList("field"), Pair.of(Concept.SKETCH_METADATA, "stats"), Pair.of(Concept.SKETCH_ESTIMATED_RESULT, "est"));
    IntStream.range(0, 1000).mapToObj(i -> RecordBox.get().add("field", i).getRecord()).forEach(countDistinct::consume);
    Clip clip = countDistinct.getResult();
    Map<String, Object> meta = clip.getMeta().asMap();
    Assert.assertEquals(meta.size(), 1);
    Assert.assertTrue(meta.containsKey("stats"));
    Map<String, Object> stats = (Map<String, Object>) meta.get("stats");
    Assert.assertEquals(stats.size(), 1);
    Assert.assertFalse((Boolean) stats.get("est"));
    Assert.assertEquals(clip.getRecords().size(), 1);
    BulletRecord actual = clip.getRecords().get(0);
    BulletRecord expected = RecordBox.get().add("myCount", 1000.0).getRecord();
    Assert.assertEquals(actual, expected);
    Assert.assertEquals(countDistinct.getRecords(), clip.getRecords());
    Assert.assertEquals(countDistinct.getMetadata().asMap(), countDistinct.getMetadata().asMap());
}

Example 55 with Clip

use of com.yahoo.bullet.result.Clip in project bullet-core by yahoo.

the class CountDistinctTest method testSingleFieldApproximateCountDistinctWithMetadata.

@Test
public void testSingleFieldApproximateCountDistinctWithMetadata() {
    BulletConfig config = makeConfiguration(4, 512);
    CountDistinct countDistinct = makeCountDistinct(config, asList("field"), Pair.of(Concept.SKETCH_METADATA, "aggregate_stats"), Pair.of(Concept.SKETCH_FAMILY, "family"), Pair.of(Concept.SKETCH_SIZE, "size"), Pair.of(Concept.SKETCH_THETA, "theta"), Pair.of(Concept.SKETCH_ESTIMATED_RESULT, "isEstimate"), Pair.of(Concept.SKETCH_STANDARD_DEVIATIONS, "stddev"));
    IntStream.range(0, 1000).mapToObj(i -> RecordBox.get().add("field", i).getRecord()).forEach(countDistinct::consume);
    Assert.assertNotNull(countDistinct.getData());
    Clip clip = countDistinct.getResult();
    Map<String, Object> meta = clip.getMeta().asMap();
    Assert.assertEquals(meta.size(), 1);
    Assert.assertTrue(meta.containsKey("aggregate_stats"));
    Map<String, Object> stats = (Map<String, Object>) meta.get("aggregate_stats");
    Assert.assertEquals(stats.size(), 5);
    Assert.assertTrue((Boolean) stats.get("isEstimate"));
    Assert.assertEquals(stats.get("family").toString(), Family.ALPHA.getFamilyName());
    int size = (Integer) stats.get("size");
    // We inserted more than 512 unique entries
    Assert.assertTrue(size > 512);
    double theta = (Double) stats.get("theta");
    Assert.assertTrue(theta <= 1.0);
    Assert.assertTrue(stats.containsKey("stddev"));
    Map<String, Map<String, Double>> standardDeviations = (Map<String, Map<String, Double>>) stats.get("stddev");
    Assert.assertEquals(standardDeviations.size(), 3);
    Assert.assertEquals(clip.getRecords().size(), 1);
    BulletRecord actual = clip.getRecords().get(0);
    double actualEstimate = (Double) actual.get(CountDistinct.DEFAULT_NEW_NAME);
    double upperOneSigma = standardDeviations.get(KMVSketch.META_STD_DEV_1).get(KMVSketch.META_STD_DEV_UB);
    double lowerOneSigma = standardDeviations.get(KMVSketch.META_STD_DEV_1).get(KMVSketch.META_STD_DEV_LB);
    double upperTwoSigma = standardDeviations.get(KMVSketch.META_STD_DEV_2).get(KMVSketch.META_STD_DEV_UB);
    double lowerTwoSigma = standardDeviations.get(KMVSketch.META_STD_DEV_2).get(KMVSketch.META_STD_DEV_LB);
    double upperThreeSigma = standardDeviations.get(KMVSketch.META_STD_DEV_3).get(KMVSketch.META_STD_DEV_UB);
    double lowerThreeSigma = standardDeviations.get(KMVSketch.META_STD_DEV_3).get(KMVSketch.META_STD_DEV_LB);
    Assert.assertTrue(actualEstimate >= lowerOneSigma);
    Assert.assertTrue(actualEstimate <= upperOneSigma);
    Assert.assertTrue(actualEstimate >= lowerTwoSigma);
    Assert.assertTrue(actualEstimate <= upperTwoSigma);
    Assert.assertTrue(actualEstimate >= lowerThreeSigma);
    Assert.assertTrue(actualEstimate <= upperThreeSigma);
    Assert.assertEquals(countDistinct.getRecords(), clip.getRecords());
    Assert.assertEquals(countDistinct.getMetadata().asMap(), countDistinct.getMetadata().asMap());
}