use of com.hortonworks.streamline.streams.layout.component.Edge in project streamline by hortonworks.
the class TestTopologyDagCreatingVisitorTest method visitSink_connectedFromProcessor.
@Test
public void visitSink_connectedFromProcessor() throws Exception {
StreamlineProcessor originProcessor = TopologyTestHelper.createStreamlineProcessor("1");
StreamlineSink originSink = TopologyTestHelper.createStreamlineSink("2");
TopologyDag originTopologyDag = new TopologyDag();
originTopologyDag.add(originProcessor);
originTopologyDag.add(originSink);
originTopologyDag.addEdge(new Edge("e1", originProcessor, originSink, "default", Stream.Grouping.SHUFFLE));
TestRunProcessor testProcessor = createTestRunProcessor(originProcessor);
TestRunSink testSink = createTestRunSink(originSink);
TestTopologyDagCreatingVisitor visitor = new TestTopologyDagCreatingVisitor(originTopologyDag, Collections.emptyMap(), Collections.singletonMap(originProcessor.getName(), testProcessor), Collections.emptyMap(), Collections.singletonMap(originSink.getName(), testSink));
visitor.visit(originProcessor);
visitor.visit(originSink);
TopologyDag testTopologyDag = visitor.getTestTopologyDag();
List<OutputComponent> testProcessors = testTopologyDag.getOutputComponents().stream().filter(o -> (o instanceof TestRunProcessor && o.getName().equals(originProcessor.getName()))).collect(toList());
List<InputComponent> testSinks = testTopologyDag.getInputComponents().stream().filter(o -> (o instanceof TestRunSink && o.getName().equals(originSink.getName()))).collect(toList());
assertEquals(1, testProcessors.size());
assertEquals(1, testSinks.size());
TestRunProcessor testRunProcessor = (TestRunProcessor) testProcessors.get(0);
assertEquals(originProcessor.getId(), testRunProcessor.getId());
TestRunSink testRunSink = (TestRunSink) testSinks.get(0);
assertEquals(originSink.getId(), testRunSink.getId());
assertEquals(testSink.getOutputFilePath(), testRunSink.getOutputFilePath());
assertEquals(1, testTopologyDag.getEdgesFrom(testProcessor).size());
assertEquals(1, testTopologyDag.getEdgesTo(testRunSink).size());
assertTrue(testTopologyDag.getEdgesFrom(testRunProcessor).get(0) == testTopologyDag.getEdgesTo(testRunSink).get(0));
}
use of com.hortonworks.streamline.streams.layout.component.Edge in project streamline by hortonworks.
the class TestTopologyDagCreatingVisitorTest method visitRulesProcessor_connectedFromSource.
@Test
public void visitRulesProcessor_connectedFromSource() throws Exception {
StreamlineSource originSource = TopologyTestHelper.createStreamlineSource("1");
RulesProcessor rulesProcessor = TopologyTestHelper.createRulesProcessor("2");
TopologyDag originTopologyDag = new TopologyDag();
originTopologyDag.add(originSource);
originTopologyDag.add(rulesProcessor);
originTopologyDag.addEdge(new Edge("e1", originSource, rulesProcessor, "default", Stream.Grouping.SHUFFLE));
TestRunSource testSource = createTestRunSource(originSource);
TestRunRulesProcessor testRulesProcessor = createTestRunRulesProcessor(rulesProcessor);
TestTopologyDagCreatingVisitor visitor = new TestTopologyDagCreatingVisitor(originTopologyDag, Collections.singletonMap(originSource.getName(), testSource), Collections.emptyMap(), Collections.singletonMap(rulesProcessor.getName(), testRulesProcessor), Collections.emptyMap());
visitor.visit(originSource);
visitor.visit(rulesProcessor);
TopologyDag testTopologyDag = visitor.getTestTopologyDag();
List<OutputComponent> testRulesProcessors = testTopologyDag.getOutputComponents().stream().filter(o -> (o instanceof TestRunRulesProcessor && o.getName().equals(rulesProcessor.getName()))).collect(toList());
List<OutputComponent> testSources = testTopologyDag.getOutputComponents().stream().filter(o -> (o instanceof TestRunSource && o.getName().equals(originSource.getName()))).collect(toList());
TestRunRulesProcessor testRunRulesProcessor = (TestRunRulesProcessor) testRulesProcessors.get(0);
TestRunSource testRunSource = (TestRunSource) testSources.get(0);
assertEquals(1, testTopologyDag.getEdgesFrom(testRunSource).size());
assertEquals(1, testTopologyDag.getEdgesTo(testRunRulesProcessor).size());
assertTrue(testTopologyDag.getEdgesFrom(testRunSource).get(0) == testTopologyDag.getEdgesTo(testRunRulesProcessor).get(0));
}
use of com.hortonworks.streamline.streams.layout.component.Edge in project streamline by hortonworks.
the class TestTopologyDagCreatingVisitorTest method visitSink_connectedFromSource.
@Test
public void visitSink_connectedFromSource() throws Exception {
StreamlineSource originSource = TopologyTestHelper.createStreamlineSource("1");
StreamlineSink originSink = TopologyTestHelper.createStreamlineSink("2");
TopologyDag originTopologyDag = new TopologyDag();
originTopologyDag.add(originSource);
originTopologyDag.add(originSink);
originTopologyDag.addEdge(new Edge("e1", originSource, originSink, "default", Stream.Grouping.SHUFFLE));
TestRunSource testSource = createTestRunSource(originSource);
TestRunSink testSink = createTestRunSink(originSink);
TestTopologyDagCreatingVisitor visitor = new TestTopologyDagCreatingVisitor(originTopologyDag, Collections.singletonMap(originSource.getName(), testSource), Collections.emptyMap(), Collections.emptyMap(), Collections.singletonMap(originSink.getName(), testSink));
visitor.visit(originSource);
visitor.visit(originSink);
TopologyDag testTopologyDag = visitor.getTestTopologyDag();
List<OutputComponent> testSources = testTopologyDag.getOutputComponents().stream().filter(o -> (o instanceof TestRunSource && o.getName().equals(originSource.getName()))).collect(toList());
List<InputComponent> testSinks = testTopologyDag.getInputComponents().stream().filter(o -> (o instanceof TestRunSink && o.getName().equals(originSink.getName()))).collect(toList());
assertEquals(1, testSinks.size());
TestRunSource testRunSource = (TestRunSource) testSources.get(0);
TestRunSink testRunSink = (TestRunSink) testSinks.get(0);
assertEquals(originSink.getId(), testRunSink.getId());
assertEquals(testSink.getOutputFilePath(), testRunSink.getOutputFilePath());
assertEquals(1, testTopologyDag.getEdgesFrom(testRunSource).size());
assertEquals(1, testTopologyDag.getEdgesTo(testRunSink).size());
assertTrue(testTopologyDag.getEdgesFrom(testRunSource).get(0) == testTopologyDag.getEdgesTo(testRunSink).get(0));
}
use of com.hortonworks.streamline.streams.layout.component.Edge in project streamline by hortonworks.
the class TestTopologyDagCreatingVisitorTest method visitRulesProcessor_connectedFromProcessor.
@Test
public void visitRulesProcessor_connectedFromProcessor() throws Exception {
StreamlineProcessor originProcessor = TopologyTestHelper.createStreamlineProcessor("1");
RulesProcessor rulesProcessor = TopologyTestHelper.createRulesProcessor("2");
TopologyDag originTopologyDag = new TopologyDag();
originTopologyDag.add(originProcessor);
originTopologyDag.add(rulesProcessor);
originTopologyDag.addEdge(new Edge("e1", originProcessor, rulesProcessor, "default", Stream.Grouping.SHUFFLE));
TestRunProcessor testProcessor = createTestRunProcessor(originProcessor);
TestRunRulesProcessor testRulesProcessor = createTestRunRulesProcessor(rulesProcessor);
TestTopologyDagCreatingVisitor visitor = new TestTopologyDagCreatingVisitor(originTopologyDag, Collections.emptyMap(), Collections.singletonMap(originProcessor.getName(), testProcessor), Collections.singletonMap(rulesProcessor.getName(), testRulesProcessor), Collections.emptyMap());
visitor.visit(originProcessor);
visitor.visit(rulesProcessor);
TopologyDag testTopologyDag = visitor.getTestTopologyDag();
List<OutputComponent> testProcessors = testTopologyDag.getOutputComponents().stream().filter(o -> (o instanceof TestRunProcessor)).collect(toList());
List<OutputComponent> testRulesProcessors = testTopologyDag.getOutputComponents().stream().filter(o -> (o instanceof TestRunRulesProcessor)).collect(toList());
Optional<OutputComponent> testRunProcessorOptional = testProcessors.stream().filter(o -> o.getName().equals(originProcessor.getName())).findAny();
Optional<OutputComponent> testRunRuleProcessorOptional = testRulesProcessors.stream().filter(o -> o.getName().equals(rulesProcessor.getName())).findAny();
assertTrue(testRunProcessorOptional.isPresent());
assertTrue(testRunRuleProcessorOptional.isPresent());
TestRunProcessor testRunProcessor = (TestRunProcessor) testRunProcessorOptional.get();
TestRunRulesProcessor testRunRuleProcessor = (TestRunRulesProcessor) testRunRuleProcessorOptional.get();
assertEquals(1, testTopologyDag.getEdgesFrom(testRunProcessor).size());
assertEquals(1, testTopologyDag.getEdgesTo(testRunRuleProcessor).size());
assertTrue(testTopologyDag.getEdgesFrom(testRunProcessor).get(0) == testTopologyDag.getEdgesTo(testRunRuleProcessor).get(0));
}
use of com.hortonworks.streamline.streams.layout.component.Edge in project streamline by hortonworks.
the class StormTopologyFluxGenerator method handleWindowedRules.
private void handleWindowedRules(RulesProcessor rulesProcessor, List<Rule> rulesWithWindow) {
// assert that RulesProcessor only has a windowed rule, not multiple rules.
if (rulesWithWindow.size() > 1) {
throw new IllegalStateException("Windowed RulesProcessor should have only one rule.");
}
Rule rule = rulesWithWindow.get(0);
Collection<Rule> rules = Collections.singletonList(rule);
Window window = rulesWithWindow.get(0).getWindow();
// create windowed bolt per unique window configuration
RulesProcessor windowedRulesProcessor = copyRulesProcessor(rulesProcessor);
windowedRulesProcessor.setRules(new ArrayList<>(rules));
windowedRulesProcessor.setId(rulesProcessor.getId());
windowedRulesProcessor.setName(rulesProcessor.getName());
windowedRulesProcessor.getConfig().setAny(RulesProcessor.CONFIG_KEY_RULES, Collections2.transform(rules, new Function<Rule, Long>() {
@Override
public Long apply(Rule input) {
return input.getId();
}
}));
LOG.debug("Rules processor with window {}", windowedRulesProcessor);
keysAndComponents.add(makeEntry(StormTopologyLayoutConstants.YAML_KEY_BOLTS, getYamlComponents(fluxComponentFactory.getFluxComponent(windowedRulesProcessor), windowedRulesProcessor)));
List<Edge> originEdgesTo = topologyDag.getEdgesTo(rulesProcessor);
List<Edge> originEdgesFrom = topologyDag.getEdgesFrom(rulesProcessor);
// remove streams before wiring
removeFluxStreamsTo(getFluxId(rulesProcessor));
removeFluxStreamsFrom(getFluxId(rulesProcessor));
// Wire the windowed bolt with the appropriate edges
wireWindowedRulesProcessor(windowedRulesProcessor, originEdgesTo, originEdgesFrom);
mayBeUpdateTopologyConfig(window);
edgeAlreadyAddedComponents.add(getFluxId(rulesProcessor));
}
Aggregations