Examples with Query com.yahoo.bullet.query.Query used on opensource projects

Example 21 with Query

use of com.yahoo.bullet.query.Query in project bullet-core by yahoo.

the class QuerierTest method testRestarting.

@Test
public void testRestarting() throws Exception {
    BulletConfig config = new BulletConfig();
    config.set(BulletConfig.RESULT_METADATA_ENABLE, true);
    config.validate();
    Query query = new Query(new Projection(), null, new Raw(null), null, WindowUtils.makeTumblingWindow(1), null);
    query.configure(config);
    Querier querier = new Querier(Querier.Mode.ALL, makeRunningQuery("", query), config);
    querier.consume(RecordBox.get().getRecord());
    Assert.assertEquals(querier.getRecords().size(), 1);
    long timeNow = System.currentTimeMillis();
    Meta meta = querier.getMetadata();
    Map<String, String> mapping = BulletConfigTest.allMetadataAsMap();
    Map<String, Object> queryMeta = (Map<String, Object>) meta.asMap().get(mapping.get(Meta.Concept.QUERY_METADATA.getName()));
    Map<String, Object> windowMeta = (Map<String, Object>) meta.asMap().get(mapping.get(Meta.Concept.WINDOW_METADATA.getName()));
    Assert.assertEquals(windowMeta.get(mapping.get(Meta.Concept.WINDOW_NUMBER.getName())), 1L);
    long startTime = (Long) queryMeta.get(mapping.get(Meta.Concept.QUERY_RECEIVE_TIME.getName()));
    long windowEmitTime = (Long) windowMeta.get(mapping.get(Meta.Concept.WINDOW_EMIT_TIME.getName()));
    Assert.assertTrue(startTime <= timeNow);
    Assert.assertTrue(windowEmitTime >= timeNow);
    Thread.sleep(1);
    querier.restart();
    Assert.assertEquals(querier.getRecords().size(), 1);
    meta = querier.getMetadata();
    queryMeta = (Map<String, Object>) meta.asMap().get(mapping.get(Meta.Concept.QUERY_METADATA.getName()));
    Assert.assertEquals(windowMeta.get(mapping.get(Meta.Concept.WINDOW_NUMBER.getName())), 1L);
    long newStartTime = (Long) queryMeta.get(mapping.get(Meta.Concept.QUERY_RECEIVE_TIME.getName()));
    // Querier#restart() no longer updates the query's start time
    Assert.assertEquals(newStartTime, startTime);
    querier.reset();
    meta = querier.getMetadata();
    windowMeta = (Map<String, Object>) meta.asMap().get(mapping.get(Meta.Concept.WINDOW_METADATA.getName()));
    long newEmitTime = (Long) windowMeta.get(mapping.get(Meta.Concept.WINDOW_EMIT_TIME.getName()));
    Assert.assertEquals(windowMeta.get(mapping.get(Meta.Concept.WINDOW_NUMBER.getName())), 2L);
    Assert.assertTrue(newEmitTime > windowEmitTime);
}

Example 22 with Query

use of com.yahoo.bullet.query.Query in project bullet-core by yahoo.

the class QuerierTest method testMetadataDisabled.

@Test
public void testMetadataDisabled() {
    BulletConfig config = new BulletConfig();
    config.set(BulletConfig.RESULT_METADATA_ENABLE, false);
    // Should clear out the default metadata
    config.validate();
    CountDistinct aggregation = new CountDistinct(Collections.singletonList("foo"), "count");
    Window window = WindowUtils.makeTumblingWindow(1);
    Query query = new Query(new Projection(), null, aggregation, null, window, null);
    query.configure(config);
    Querier querier = make(Querier.Mode.PARTITION, query, config);
    querier.consume(RecordBox.get().add("foo", "A").getRecord());
    Assert.assertTrue(querier.getMetadata().asMap().isEmpty());
}

Example 23 with Query

use of com.yahoo.bullet.query.Query in project bullet-core by yahoo.

the class QuerierTest method testRateLimiting.

@Test
public void testRateLimiting() throws Exception {
    BulletConfig config = new BulletConfig();
    config.set(BulletConfig.RATE_LIMIT_ENABLE, true);
    config.set(BulletConfig.RATE_LIMIT_TIME_INTERVAL, 1);
    config.set(BulletConfig.RATE_LIMIT_MAX_EMIT_COUNT, 1);
    config.validate();
    Query query = makeRawQuery();
    query.configure(config);
    Querier querier = make(Querier.Mode.ALL, "", query, config);
    Assert.assertFalse(querier.isExceedingRateLimit());
    Assert.assertNull(querier.getRateLimitError());
    IntStream.range(0, 1000).forEach(i -> querier.getRecords());
    // To make sure it's time to check again
    Thread.sleep(1);
    Assert.assertTrue(querier.isExceedingRateLimit());
    Assert.assertNotNull(querier.getRateLimitError());
}

Example 24 with Query

use of com.yahoo.bullet.query.Query in project bullet-core by yahoo.

the class QueryManagerTest method testCategorizingPartitioning.

@Test
public void testCategorizingPartitioning() {
    QueryManager manager = new QueryManager(getEqualityPartitionerConfig("A", "B"));
    Query queryA = getQuery(ImmutablePair.of("A", "foo"));
    Query queryB = getQuery(ImmutablePair.of("A", "foo"), ImmutablePair.of("B", "bar"));
    Querier querierA = getQuerier(queryA);
    Querier querierB = getQuerier(queryB);
    manager.addQuery("idA", querierA);
    manager.addQuery("idB", querierB);
    BulletRecord recordA = RecordBox.get().add("A", "foo").add("B", "bar").getRecord();
    BulletRecord recordB = RecordBox.get().add("A", "foo").getRecord();
    QueryCategorizer categorizer = manager.categorize(recordA);
    Assert.assertEquals(categorizer.getDone().size(), 0);
    Assert.assertEquals(categorizer.getClosed().size(), 0);
    Assert.assertEquals(categorizer.getRateLimited().size(), 0);
    verify(querierA, times(1)).consume(recordA);
    verify(querierB, times(1)).consume(recordA);
    categorizer = manager.categorize(recordB);
    Assert.assertEquals(categorizer.getDone().size(), 0);
    Assert.assertEquals(categorizer.getClosed().size(), 0);
    Assert.assertEquals(categorizer.getRateLimited().size(), 0);
    verify(querierA, times(1)).consume(recordB);
    verify(querierB, never()).consume(recordB);
}

Example 25 with Query

use of com.yahoo.bullet.query.Query in project bullet-core by yahoo.

the class QueryManagerTest method testNoPartitioning.

@Test
public void testNoPartitioning() {
    QueryManager manager = new QueryManager(new BulletConfig());
    Query queryA = getQuery(ImmutablePair.of("A", "foo"));
    Query queryB = getQuery(ImmutablePair.of("A", "foo"), ImmutablePair.of("B", "bar"));
    Query queryC = getQuery();
    Querier querierA = getQuerier(queryA);
    Querier querierB = getQuerier(queryB);
    Querier querierC = getQuerier(queryC);
    manager.addQuery("idA", querierA);
    manager.addQuery("idB", querierB);
    manager.addQuery("idC", querierC);
    BulletRecord recordA = RecordBox.get().add("A", "foo").getRecord();
    BulletRecord recordB = RecordBox.get().add("A", "foo").add("B", "bar").getRecord();
    BulletRecord recordC = RecordBox.get().getRecord();
    Map<String, Querier> partitionA = manager.partition(recordA);
    Map<String, Querier> partitionB = manager.partition(recordB);
    Map<String, Querier> partitionC = manager.partition(recordC);
    Assert.assertEquals(partitionA.size(), 3);
    Assert.assertSame(partitionA.get("idA"), querierA);
    Assert.assertSame(partitionA.get("idB"), querierB);
    Assert.assertSame(partitionA.get("idC"), querierC);
    Assert.assertEquals(partitionB.size(), 3);
    Assert.assertSame(partitionB.get("idA"), querierA);
    Assert.assertSame(partitionB.get("idB"), querierB);
    Assert.assertSame(partitionB.get("idC"), querierC);
    Assert.assertEquals(partitionC.size(), 3);
    Assert.assertSame(partitionC.get("idA"), querierA);
    Assert.assertSame(partitionC.get("idB"), querierB);
    Assert.assertSame(partitionC.get("idC"), querierC);
}