Examples with Config com.twitter.heron.api.Config used on opensource projects

Example 51 with Config

use of com.twitter.heron.api.Config in project incubator-heron by apache.

the class WindowManagerTest method testExpireThreshold.

@Test
public void testExpireThreshold() throws Exception {
    int threshold = WindowManager.EXPIRE_EVENTS_THRESHOLD;
    int windowLength = 5;
    CountEvictionPolicy<Integer> countEvictionPolicy = new CountEvictionPolicy<Integer>(5);
    windowManager.setEvictionPolicy(countEvictionPolicy);
    TriggerPolicy<Integer, ?> triggerPolicy = new TimeTriggerPolicy<Integer>(Duration.ofHours(1).toMillis());
    triggerPolicy.setEvictionPolicy(countEvictionPolicy);
    triggerPolicy.setTriggerHandler(windowManager);
    triggerPolicy.setTopologyConfig(new Config());
    triggerPolicy.start();
    windowManager.setTriggerPolicy(triggerPolicy);
    for (int i : seq(1, 5)) {
        windowManager.add(i);
    }
    // nothing expired yet
    assertTrue(listener.onExpiryEvents.isEmpty());
    for (int i : seq(6, 10)) {
        windowManager.add(i);
    }
    for (int i : seq(11, threshold)) {
        windowManager.add(i);
    }
    // window should be compacted and events should be expired.
    assertEquals(seq(1, threshold - windowLength), listener.onExpiryEvents);
}

Example 52 with Config

use of com.twitter.heron.api.Config in project incubator-heron by apache.

the class GeneralReduceByKeyAndWindowOperatorTest method getReduceByWindowOperator.

@SuppressWarnings({ "rawtypes", "unchecked" })
private GeneralReduceByKeyAndWindowOperator<String, KeyValue<String, Integer>, Integer> getReduceByWindowOperator(Integer identity) {
    GeneralReduceByKeyAndWindowOperator<String, KeyValue<String, Integer>, Integer> reduceByWindowOperator = new GeneralReduceByKeyAndWindowOperator<>(x -> x.getKey(), identity, (o, o2) -> o + o2.getValue());
    reduceByWindowOperator.prepare(new Config(), PowerMockito.mock(TopologyContext.class), new OutputCollector(new IOutputCollector() {

        @Override
        public void reportError(Throwable error) {
        }

        @Override
        public List<Integer> emit(String streamId, Collection<Tuple> anchors, List<Object> tuple) {
            emittedTuples.addAll(tuple);
            return null;
        }

        @Override
        public void emitDirect(int taskId, String streamId, Collection<Tuple> anchors, List<Object> tuple) {
        }

        @Override
        public void ack(Tuple input) {
        }

        @Override
        public void fail(Tuple input) {
        }
    }));
    return reduceByWindowOperator;
}

Example 53 with Config

use of com.twitter.heron.api.Config in project incubator-heron by apache.

the class ReduceByKeyAndWindowOperatorTest method getReduceByWindowOperator.

@SuppressWarnings({ "rawtypes", "unchecked" })
private ReduceByKeyAndWindowOperator<String, Integer, String> getReduceByWindowOperator() {
    ReduceByKeyAndWindowOperator<String, Integer, String> reduceByWindowOperator = new ReduceByKeyAndWindowOperator<>(x -> x, x -> 1, (o, o2) -> o + o2);
    reduceByWindowOperator.prepare(new Config(), PowerMockito.mock(TopologyContext.class), new OutputCollector(new IOutputCollector() {

        @Override
        public void reportError(Throwable error) {
        }

        @Override
        public List<Integer> emit(String streamId, Collection<Tuple> anchors, List<Object> tuple) {
            emittedTuples.addAll(tuple);
            return null;
        }

        @Override
        public void emitDirect(int taskId, String streamId, Collection<Tuple> anchors, List<Object> tuple) {
        }

        @Override
        public void ack(Tuple input) {
        }

        @Override
        public void fail(Tuple input) {
        }
    }));
    return reduceByWindowOperator;
}

Example 54 with Config

use of com.twitter.heron.api.Config in project incubator-heron by apache.

the class TopologyUtilsTest method testGetComponentParallelism.

@Test
public void testGetComponentParallelism() {
    int componentParallelism = 4;
    Config topologyConfig = new Config();
    Map<String, Integer> spouts = new HashMap<>();
    spouts.put("spout", componentParallelism);
    Map<String, Integer> bolts = new HashMap<>();
    bolts.put("bolt", componentParallelism);
    TopologyAPI.Topology topology = TopologyTests.createTopology("testTopology", topologyConfig, spouts, bolts);
    Map<String, Integer> componentParallelismMap = TopologyUtils.getComponentParallelism(topology);
    Assert.assertEquals(componentParallelism, componentParallelismMap.get("spout").intValue());
    Assert.assertEquals(componentParallelism, componentParallelismMap.get("bolt").intValue());
}

Example 55 with Config

use of com.twitter.heron.api.Config in project incubator-heron by apache.

the class TopologyUtilsTest method testGetComponentCpuMapSomeCpuSpecified.

@Test
public void testGetComponentCpuMapSomeCpuSpecified() {
    int componentParallelism = 2;
    Config topologyConfig = new Config();
    Map<String, Integer> spouts = new HashMap<>();
    spouts.put("spout", componentParallelism);
    Map<String, Integer> bolts = new HashMap<>();
    bolts.put("bolt", componentParallelism);
    double spoutCpu = 2.0f;
    topologyConfig.setComponentCpu("spout", spoutCpu);
    // sort the component cpu map
    Map<String, Double> cpuMap = new TreeMap<>(TopologyUtils.getComponentCpuMapConfig(TopologyTests.createTopology("test", topologyConfig, spouts, bolts)));
    // Component cpu map sets only spout's cpu
    Assert.assertArrayEquals(new String[] { "spout" }, cpuMap.keySet().toArray());
    Assert.assertArrayEquals(new Double[] { spoutCpu }, cpuMap.values().toArray());
}