Examples with RulesProcessor com.hortonworks.streamline.streams.layout.component.impl.RulesProcessor used on opensource projects

Example 6 with RulesProcessor

use of com.hortonworks.streamline.streams.layout.component.impl.RulesProcessor in project streamline by hortonworks.

the class RulesTopologyTest method createTopology.

protected StormTopology createTopology() {
    TopologyBuilder builder = new TopologyBuilder();
    RulesProcessor rulesProcessor = createRulesProcessor();
    builder.setSpout(RULES_TEST_SPOUT, new RulesTestSpout());
    builder.setBolt(RULES_BOLT, createRulesBolt(rulesProcessor, getScriptType())).shuffleGrouping(RULES_TEST_SPOUT);
    builder.setBolt(RULES_TEST_SINK_BOLT_1, new RulesTestSinkBolt()).shuffleGrouping(RULES_BOLT, rulesProcessor.getRules().get(0).getOutputStreamNameForAction(rulesProcessor.getRules().get(0).getActions().iterator().next()));
    builder.setBolt(RULES_TEST_SINK_BOLT_2, new RulesTestSinkBolt()).shuffleGrouping(RULES_BOLT, rulesProcessor.getRules().get(1).getOutputStreamNameForAction(rulesProcessor.getRules().get(1).getActions().iterator().next()));
    return builder.createTopology();
}

Example 7 with RulesProcessor

use of com.hortonworks.streamline.streams.layout.component.impl.RulesProcessor in project streamline by hortonworks.

the class TestTopologyDagCreatingVisitorTest method visitRulesProcessor_connectedFromSource.

@Test
public void visitRulesProcessor_connectedFromSource() throws Exception {
    StreamlineSource originSource = TopologyTestHelper.createStreamlineSource("1");
    RulesProcessor rulesProcessor = TopologyTestHelper.createRulesProcessor("2");
    TopologyDag originTopologyDag = new TopologyDag();
    originTopologyDag.add(originSource);
    originTopologyDag.add(rulesProcessor);
    originTopologyDag.addEdge(new Edge("e1", originSource, rulesProcessor, "default", Stream.Grouping.SHUFFLE));
    TestRunSource testSource = createTestRunSource(originSource);
    TestRunRulesProcessor testRulesProcessor = createTestRunRulesProcessor(rulesProcessor);
    TestTopologyDagCreatingVisitor visitor = new TestTopologyDagCreatingVisitor(originTopologyDag, Collections.singletonMap(originSource.getName(), testSource), Collections.emptyMap(), Collections.singletonMap(rulesProcessor.getName(), testRulesProcessor), Collections.emptyMap());
    visitor.visit(originSource);
    visitor.visit(rulesProcessor);
    TopologyDag testTopologyDag = visitor.getTestTopologyDag();
    List<OutputComponent> testRulesProcessors = testTopologyDag.getOutputComponents().stream().filter(o -> (o instanceof TestRunRulesProcessor && o.getName().equals(rulesProcessor.getName()))).collect(toList());
    List<OutputComponent> testSources = testTopologyDag.getOutputComponents().stream().filter(o -> (o instanceof TestRunSource && o.getName().equals(originSource.getName()))).collect(toList());
    TestRunRulesProcessor testRunRulesProcessor = (TestRunRulesProcessor) testRulesProcessors.get(0);
    TestRunSource testRunSource = (TestRunSource) testSources.get(0);
    assertEquals(1, testTopologyDag.getEdgesFrom(testRunSource).size());
    assertEquals(1, testTopologyDag.getEdgesTo(testRunRulesProcessor).size());
    assertTrue(testTopologyDag.getEdgesFrom(testRunSource).get(0) == testTopologyDag.getEdgesTo(testRunRulesProcessor).get(0));
}

Example 8 with RulesProcessor

use of com.hortonworks.streamline.streams.layout.component.impl.RulesProcessor in project streamline by hortonworks.

the class TestTopologyDagCreatingVisitorTest method visitRulesProcessor_connectedFromProcessor.

@Test
public void visitRulesProcessor_connectedFromProcessor() throws Exception {
    StreamlineProcessor originProcessor = TopologyTestHelper.createStreamlineProcessor("1");
    RulesProcessor rulesProcessor = TopologyTestHelper.createRulesProcessor("2");
    TopologyDag originTopologyDag = new TopologyDag();
    originTopologyDag.add(originProcessor);
    originTopologyDag.add(rulesProcessor);
    originTopologyDag.addEdge(new Edge("e1", originProcessor, rulesProcessor, "default", Stream.Grouping.SHUFFLE));
    TestRunProcessor testProcessor = createTestRunProcessor(originProcessor);
    TestRunRulesProcessor testRulesProcessor = createTestRunRulesProcessor(rulesProcessor);
    TestTopologyDagCreatingVisitor visitor = new TestTopologyDagCreatingVisitor(originTopologyDag, Collections.emptyMap(), Collections.singletonMap(originProcessor.getName(), testProcessor), Collections.singletonMap(rulesProcessor.getName(), testRulesProcessor), Collections.emptyMap());
    visitor.visit(originProcessor);
    visitor.visit(rulesProcessor);
    TopologyDag testTopologyDag = visitor.getTestTopologyDag();
    List<OutputComponent> testProcessors = testTopologyDag.getOutputComponents().stream().filter(o -> (o instanceof TestRunProcessor)).collect(toList());
    List<OutputComponent> testRulesProcessors = testTopologyDag.getOutputComponents().stream().filter(o -> (o instanceof TestRunRulesProcessor)).collect(toList());
    Optional<OutputComponent> testRunProcessorOptional = testProcessors.stream().filter(o -> o.getName().equals(originProcessor.getName())).findAny();
    Optional<OutputComponent> testRunRuleProcessorOptional = testRulesProcessors.stream().filter(o -> o.getName().equals(rulesProcessor.getName())).findAny();
    assertTrue(testRunProcessorOptional.isPresent());
    assertTrue(testRunRuleProcessorOptional.isPresent());
    TestRunProcessor testRunProcessor = (TestRunProcessor) testRunProcessorOptional.get();
    TestRunRulesProcessor testRunRuleProcessor = (TestRunRulesProcessor) testRunRuleProcessorOptional.get();
    assertEquals(1, testTopologyDag.getEdgesFrom(testRunProcessor).size());
    assertEquals(1, testTopologyDag.getEdgesTo(testRunRuleProcessor).size());
    assertTrue(testTopologyDag.getEdgesFrom(testRunProcessor).get(0) == testTopologyDag.getEdgesTo(testRunRuleProcessor).get(0));
}

Example 9 with RulesProcessor

use of com.hortonworks.streamline.streams.layout.component.impl.RulesProcessor in project streamline by hortonworks.

the class WindowRulesBoltTest method testCountBasedWindowWithGroupbyUnordered.

@Test
public void testCountBasedWindowWithGroupbyUnordered() throws Exception {
    String rulesJson = readFile("/window-rule-groupby-unordered.json");
    RulesProcessor rulesProcessor = Utils.createObjectFromJson(rulesJson, RulesProcessor.class);
    Window windowConfig = rulesProcessor.getRules().get(0).getWindow();
    WindowRulesBolt wb = new WindowRulesBolt(rulesJson, RuleProcessorRuntime.ScriptType.SQL);
    wb.withWindowConfig(windowConfig);
    WindowedBoltExecutor wbe = new WindowedBoltExecutor(wb);
    Map<String, Object> conf = wb.getComponentConfiguration();
    wbe.prepare(conf, mockContext, mockCollector);
    wbe.execute(getNextTuple(10));
    wbe.execute(getNextTuple(15));
    wbe.execute(getNextTuple(11));
    wbe.execute(getNextTuple(16));
    new Verifications() {

        {
            String streamId;
            Collection<Tuple> anchors;
            List<List<Object>> tuples = new ArrayList<>();
            mockCollector.emit(streamId = withCapture(), anchors = withCapture(), withCapture(tuples));
            Assert.assertEquals(2, tuples.size());
            Map<String, Object> fieldsAndValues = ((StreamlineEvent) tuples.get(0).get(0));
            Assert.assertEquals(2, fieldsAndValues.get("deptid"));
            Assert.assertEquals(110, fieldsAndValues.get("salary_MAX"));
            fieldsAndValues = ((StreamlineEvent) tuples.get(1).get(0));
            Assert.assertEquals(3, fieldsAndValues.get("deptid"));
            Assert.assertEquals(160, fieldsAndValues.get("salary_MAX"));
        }
    };
}

Example 10 with RulesProcessor

use of com.hortonworks.streamline.streams.layout.component.impl.RulesProcessor in project streamline by hortonworks.

the class StormTopologyFluxGenerator method handleWindowedRules.

private void handleWindowedRules(RulesProcessor rulesProcessor, List<Rule> rulesWithWindow) {
    // assert that RulesProcessor only has a windowed rule, not multiple rules.
    if (rulesWithWindow.size() > 1) {
        throw new IllegalStateException("Windowed RulesProcessor should have only one rule.");
    }
    Rule rule = rulesWithWindow.get(0);
    Collection<Rule> rules = Collections.singletonList(rule);
    Window window = rulesWithWindow.get(0).getWindow();
    // create windowed bolt per unique window configuration
    RulesProcessor windowedRulesProcessor = copyRulesProcessor(rulesProcessor);
    windowedRulesProcessor.setRules(new ArrayList<>(rules));
    windowedRulesProcessor.setId(rulesProcessor.getId());
    windowedRulesProcessor.setName(rulesProcessor.getName());
    windowedRulesProcessor.getConfig().setAny(RulesProcessor.CONFIG_KEY_RULES, Collections2.transform(rules, new Function<Rule, Long>() {

        @Override
        public Long apply(Rule input) {
            return input.getId();
        }
    }));
    LOG.debug("Rules processor with window {}", windowedRulesProcessor);
    keysAndComponents.add(makeEntry(StormTopologyLayoutConstants.YAML_KEY_BOLTS, getYamlComponents(fluxComponentFactory.getFluxComponent(windowedRulesProcessor), windowedRulesProcessor)));
    List<Edge> originEdgesTo = topologyDag.getEdgesTo(rulesProcessor);
    List<Edge> originEdgesFrom = topologyDag.getEdgesFrom(rulesProcessor);
    // remove streams before wiring
    removeFluxStreamsTo(getFluxId(rulesProcessor));
    removeFluxStreamsFrom(getFluxId(rulesProcessor));
    // Wire the windowed bolt with the appropriate edges
    wireWindowedRulesProcessor(windowedRulesProcessor, originEdgesTo, originEdgesFrom);
    mayBeUpdateTopologyConfig(window);
    edgeAlreadyAddedComponents.add(getFluxId(rulesProcessor));
}