Examples with Config org.apache.heron.api.Config used on opensource projects

Example 6 with Config

use of org.apache.heron.api.Config in project heron by twitter.

the class WindowManagerTest method testTimeBasedWindow.

@Test
public void testTimeBasedWindow() throws Exception {
    EvictionPolicy<Integer, ?> evictionPolicy = new TimeEvictionPolicy<Integer>(Duration.ofSeconds(1).toMillis());
    windowManager.setEvictionPolicy(evictionPolicy);
    /*
         * Don't wait for Timetrigger to fire since this could lead to timing issues in unit tests.
         * Set it to a large value and trigger manually.
          */
    TriggerPolicy<Integer, ?> triggerPolicy = new TimeTriggerPolicy<Integer>(Duration.ofDays(1).toMillis());
    triggerPolicy.setTriggerHandler(windowManager);
    triggerPolicy.setEvictionPolicy(evictionPolicy);
    triggerPolicy.setTopologyConfig(new Config());
    triggerPolicy.start();
    windowManager.setTriggerPolicy(triggerPolicy);
    long now = System.currentTimeMillis();
    // add with past ts
    for (int i : seq(1, 50)) {
        windowManager.add(i, now - 1000);
    }
    // add with current ts
    for (int i : seq(51, WindowManager.EXPIRE_EVENTS_THRESHOLD)) {
        windowManager.add(i, now);
    }
    // first 50 should have expired due to expire events threshold
    assertEquals(50, listener.onExpiryEvents.size());
    // add more events with past ts
    for (int i : seq(WindowManager.EXPIRE_EVENTS_THRESHOLD + 1, WindowManager.EXPIRE_EVENTS_THRESHOLD + 100)) {
        windowManager.add(i, now - 1000);
    }
    // simulate the time trigger by setting the reference time and invoking onTrigger() manually
    evictionPolicy.setContext(new DefaultEvictionContext(now + 100));
    windowManager.onTrigger();
    // 100 events with past ts should expire
    assertEquals(100, listener.onExpiryEvents.size());
    assertEquals(seq(WindowManager.EXPIRE_EVENTS_THRESHOLD + 1, WindowManager.EXPIRE_EVENTS_THRESHOLD + 100), listener.onExpiryEvents);
    List<Integer> activationsEvents = seq(51, WindowManager.EXPIRE_EVENTS_THRESHOLD);
    assertEquals(seq(51, WindowManager.EXPIRE_EVENTS_THRESHOLD), listener.onActivationEvents);
    assertEquals(seq(51, WindowManager.EXPIRE_EVENTS_THRESHOLD), listener.onActivationNewEvents);
    // activation expired list should contain even the ones expired due to EXPIRE_EVENTS_THRESHOLD
    List<Integer> expiredList = seq(1, 50);
    expiredList.addAll(seq(WindowManager.EXPIRE_EVENTS_THRESHOLD + 1, WindowManager.EXPIRE_EVENTS_THRESHOLD + 100));
    assertEquals(expiredList, listener.onActivationExpiredEvents);
    listener.clear();
    // add more events with current ts
    List<Integer> newEvents = seq(WindowManager.EXPIRE_EVENTS_THRESHOLD + 101, WindowManager.EXPIRE_EVENTS_THRESHOLD + 200);
    for (int i : newEvents) {
        windowManager.add(i, now);
    }
    activationsEvents.addAll(newEvents);
    // simulate the time trigger by setting the reference time and invoking onTrigger() manually
    evictionPolicy.setContext(new DefaultEvictionContext(now + 200));
    windowManager.onTrigger();
    assertTrue(listener.onExpiryEvents.isEmpty());
    assertEquals(activationsEvents, listener.onActivationEvents);
    assertEquals(newEvents, listener.onActivationNewEvents);
}

Example 7 with Config

use of org.apache.heron.api.Config in project heron by twitter.

the class ReduceByKeyAndWindowOperatorTest method getReduceByWindowOperator.

@SuppressWarnings({ "rawtypes", "unchecked" })
private ReduceByKeyAndWindowOperator<String, String, Integer> getReduceByWindowOperator() {
    ReduceByKeyAndWindowOperator<String, String, Integer> reduceByWindowOperator = new ReduceByKeyAndWindowOperator<>(x -> x, x -> 1, (o, o2) -> o + o2);
    reduceByWindowOperator.prepare(new Config(), PowerMockito.mock(TopologyContext.class), new OutputCollector(new IOutputCollector() {

        @Override
        public void reportError(Throwable error) {
        }

        @Override
        public List<Integer> emit(String streamId, Collection<Tuple> anchors, List<Object> tuple) {
            emittedTuples.addAll(tuple);
            return null;
        }

        @Override
        public void emitDirect(int taskId, String streamId, Collection<Tuple> anchors, List<Object> tuple) {
        }

        @Override
        public void ack(Tuple input) {
        }

        @Override
        public void fail(Tuple input) {
        }
    }));
    return reduceByWindowOperator;
}

Example 8 with Config

use of org.apache.heron.api.Config in project heron by twitter.

the class Utils method getConfigBuilder.

/**
 * Converts a Heron Config object into a TopologyAPI.Config.Builder. Config entries with null
 * keys or values are ignored.
 *
 * @param config heron Config object
 * @return TopologyAPI.Config.Builder with values loaded from config
 */
public static TopologyAPI.Config.Builder getConfigBuilder(Config config) {
    TopologyAPI.Config.Builder cBldr = TopologyAPI.Config.newBuilder();
    Set<String> apiVars = config.getApiVars();
    for (String key : config.keySet()) {
        if (key == null) {
            LOG.warning("ignore: null config key found");
            continue;
        }
        Object value = config.get(key);
        if (value == null) {
            LOG.warning("ignore: config key " + key + " has null value");
            continue;
        }
        TopologyAPI.Config.KeyValue.Builder b = TopologyAPI.Config.KeyValue.newBuilder();
        b.setKey(key);
        if (apiVars.contains(key)) {
            b.setType(TopologyAPI.ConfigValueType.STRING_VALUE);
            b.setValue(value.toString());
        } else {
            b.setType(TopologyAPI.ConfigValueType.JAVA_SERIALIZED_VALUE);
            b.setSerializedValue(ByteString.copyFrom(serialize(value)));
        }
        cBldr.addKvs(b);
    }
    return cBldr;
}

Example 9 with Config

use of org.apache.heron.api.Config in project heron by twitter.

the class TopologyUtilsTest method testGetTotalInstance.

@Test
public void testGetTotalInstance() {
    int componentParallelism = 4;
    Config topologyConfig = new Config();
    Map<String, Integer> spouts = new HashMap<>();
    spouts.put("spout", componentParallelism);
    Map<String, Integer> bolts = new HashMap<>();
    bolts.put("bolt", componentParallelism);
    TopologyAPI.Topology topology = TopologyTests.createTopology("testTopology", topologyConfig, spouts, bolts);
    Assert.assertEquals((spouts.size() + bolts.size()) * componentParallelism, TopologyUtils.getTotalInstance(topology));
}

Example 10 with Config

use of org.apache.heron.api.Config in project heron by twitter.

the class TopologyUtilsTest method testGetComponentCpuMapAllCpuSpecified.

@Test
public void testGetComponentCpuMapAllCpuSpecified() {
    int componentParallelism = 2;
    Config topologyConfig = new Config();
    Map<String, Integer> spouts = new HashMap<>();
    spouts.put("spout", componentParallelism);
    Map<String, Integer> bolts = new HashMap<>();
    bolts.put("bolt", componentParallelism);
    double boltCpu = 1.0;
    double spoutCpu = 2.0;
    topologyConfig.setComponentCpu("spout", spoutCpu);
    topologyConfig.setComponentCpu("bolt", boltCpu);
    // sort the component CPU map
    Map<String, Double> cpuMap = new TreeMap<>(TopologyUtils.getComponentCpuMapConfig(TopologyTests.createTopology("test", topologyConfig, spouts, bolts)));
    Assert.assertArrayEquals(new String[] { "bolt", "spout" }, cpuMap.keySet().toArray());
    Assert.assertArrayEquals(new Double[] { boltCpu, spoutCpu }, cpuMap.values().toArray());
}