Examples with StreamlineProcessor com.hortonworks.streamline.streams.layout.component.StreamlineProcessor used on opensource projects

Example 11 with StreamlineProcessor

use of com.hortonworks.streamline.streams.layout.component.StreamlineProcessor in project streamline by hortonworks.

the class TestTopologyDagCreatingVisitorTest method visitProcessor_connectedFromProcessor.

@Test
public void visitProcessor_connectedFromProcessor() throws Exception {
    StreamlineProcessor originProcessor = TopologyTestHelper.createStreamlineProcessor("1");
    StreamlineProcessor originProcessor2 = TopologyTestHelper.createStreamlineProcessor("2");
    TopologyDag originTopologyDag = new TopologyDag();
    originTopologyDag.add(originProcessor);
    originTopologyDag.add(originProcessor2);
    originTopologyDag.addEdge(new Edge("e1", originProcessor, originProcessor2, "default", Stream.Grouping.SHUFFLE));
    TestRunProcessor testProcessor = createTestRunProcessor(originProcessor);
    TestRunProcessor testProcessor2 = createTestRunProcessor(originProcessor2);
    Map<String, TestRunProcessor> processorMap = new HashMap<>();
    processorMap.put(originProcessor.getName(), testProcessor);
    processorMap.put(originProcessor2.getName(), testProcessor2);
    TestTopologyDagCreatingVisitor visitor = new TestTopologyDagCreatingVisitor(originTopologyDag, Collections.emptyMap(), processorMap, Collections.emptyMap(), Collections.emptyMap());
    visitor.visit(originProcessor);
    visitor.visit(originProcessor2);
    TopologyDag testTopologyDag = visitor.getTestTopologyDag();
    List<OutputComponent> testProcessors = testTopologyDag.getOutputComponents().stream().filter(o -> (o instanceof TestRunProcessor)).collect(toList());
    Optional<OutputComponent> testRunProcessorOptional = testProcessors.stream().filter(o -> o.getName().equals(originProcessor.getName())).findAny();
    Optional<OutputComponent> testRunProcessor2Optional = testProcessors.stream().filter(o -> o.getName().equals(originProcessor2.getName())).findAny();
    assertTrue(testRunProcessorOptional.isPresent());
    assertTrue(testRunProcessor2Optional.isPresent());
    TestRunProcessor testRunProcessor = (TestRunProcessor) testRunProcessorOptional.get();
    TestRunProcessor testRunProcessor2 = (TestRunProcessor) testRunProcessor2Optional.get();
    assertEquals(1, testTopologyDag.getEdgesFrom(testRunProcessor).size());
    assertEquals(1, testTopologyDag.getEdgesTo(testRunProcessor2).size());
    assertTrue(testTopologyDag.getEdgesFrom(testRunProcessor).get(0) == testTopologyDag.getEdgesTo(testRunProcessor2).get(0));
}

Example 12 with StreamlineProcessor

use of com.hortonworks.streamline.streams.layout.component.StreamlineProcessor in project streamline by hortonworks.

the class TestTopologyDagCreatingVisitorTest method visitSink_connectedFromProcessor.

@Test
public void visitSink_connectedFromProcessor() throws Exception {
    StreamlineProcessor originProcessor = TopologyTestHelper.createStreamlineProcessor("1");
    StreamlineSink originSink = TopologyTestHelper.createStreamlineSink("2");
    TopologyDag originTopologyDag = new TopologyDag();
    originTopologyDag.add(originProcessor);
    originTopologyDag.add(originSink);
    originTopologyDag.addEdge(new Edge("e1", originProcessor, originSink, "default", Stream.Grouping.SHUFFLE));
    TestRunProcessor testProcessor = createTestRunProcessor(originProcessor);
    TestRunSink testSink = createTestRunSink(originSink);
    TestTopologyDagCreatingVisitor visitor = new TestTopologyDagCreatingVisitor(originTopologyDag, Collections.emptyMap(), Collections.singletonMap(originProcessor.getName(), testProcessor), Collections.emptyMap(), Collections.singletonMap(originSink.getName(), testSink));
    visitor.visit(originProcessor);
    visitor.visit(originSink);
    TopologyDag testTopologyDag = visitor.getTestTopologyDag();
    List<OutputComponent> testProcessors = testTopologyDag.getOutputComponents().stream().filter(o -> (o instanceof TestRunProcessor && o.getName().equals(originProcessor.getName()))).collect(toList());
    List<InputComponent> testSinks = testTopologyDag.getInputComponents().stream().filter(o -> (o instanceof TestRunSink && o.getName().equals(originSink.getName()))).collect(toList());
    assertEquals(1, testProcessors.size());
    assertEquals(1, testSinks.size());
    TestRunProcessor testRunProcessor = (TestRunProcessor) testProcessors.get(0);
    assertEquals(originProcessor.getId(), testRunProcessor.getId());
    TestRunSink testRunSink = (TestRunSink) testSinks.get(0);
    assertEquals(originSink.getId(), testRunSink.getId());
    assertEquals(testSink.getOutputFilePath(), testRunSink.getOutputFilePath());
    assertEquals(1, testTopologyDag.getEdgesFrom(testProcessor).size());
    assertEquals(1, testTopologyDag.getEdgesTo(testRunSink).size());
    assertTrue(testTopologyDag.getEdgesFrom(testRunProcessor).get(0) == testTopologyDag.getEdgesTo(testRunSink).get(0));
}

Example 13 with StreamlineProcessor

use of com.hortonworks.streamline.streams.layout.component.StreamlineProcessor in project streamline by hortonworks.

the class TestTopologyDagCreatingVisitorTest method visitRulesProcessor_connectedFromProcessor.

@Test
public void visitRulesProcessor_connectedFromProcessor() throws Exception {
    StreamlineProcessor originProcessor = TopologyTestHelper.createStreamlineProcessor("1");
    RulesProcessor rulesProcessor = TopologyTestHelper.createRulesProcessor("2");
    TopologyDag originTopologyDag = new TopologyDag();
    originTopologyDag.add(originProcessor);
    originTopologyDag.add(rulesProcessor);
    originTopologyDag.addEdge(new Edge("e1", originProcessor, rulesProcessor, "default", Stream.Grouping.SHUFFLE));
    TestRunProcessor testProcessor = createTestRunProcessor(originProcessor);
    TestRunRulesProcessor testRulesProcessor = createTestRunRulesProcessor(rulesProcessor);
    TestTopologyDagCreatingVisitor visitor = new TestTopologyDagCreatingVisitor(originTopologyDag, Collections.emptyMap(), Collections.singletonMap(originProcessor.getName(), testProcessor), Collections.singletonMap(rulesProcessor.getName(), testRulesProcessor), Collections.emptyMap());
    visitor.visit(originProcessor);
    visitor.visit(rulesProcessor);
    TopologyDag testTopologyDag = visitor.getTestTopologyDag();
    List<OutputComponent> testProcessors = testTopologyDag.getOutputComponents().stream().filter(o -> (o instanceof TestRunProcessor)).collect(toList());
    List<OutputComponent> testRulesProcessors = testTopologyDag.getOutputComponents().stream().filter(o -> (o instanceof TestRunRulesProcessor)).collect(toList());
    Optional<OutputComponent> testRunProcessorOptional = testProcessors.stream().filter(o -> o.getName().equals(originProcessor.getName())).findAny();
    Optional<OutputComponent> testRunRuleProcessorOptional = testRulesProcessors.stream().filter(o -> o.getName().equals(rulesProcessor.getName())).findAny();
    assertTrue(testRunProcessorOptional.isPresent());
    assertTrue(testRunRuleProcessorOptional.isPresent());
    TestRunProcessor testRunProcessor = (TestRunProcessor) testRunProcessorOptional.get();
    TestRunRulesProcessor testRunRuleProcessor = (TestRunRulesProcessor) testRunRuleProcessorOptional.get();
    assertEquals(1, testTopologyDag.getEdgesFrom(testRunProcessor).size());
    assertEquals(1, testTopologyDag.getEdgesTo(testRunRuleProcessor).size());
    assertTrue(testTopologyDag.getEdgesFrom(testRunProcessor).get(0) == testTopologyDag.getEdgesTo(testRunRuleProcessor).get(0));
}

Example 14 with StreamlineProcessor

use of com.hortonworks.streamline.streams.layout.component.StreamlineProcessor in project streamline by hortonworks.

the class TopologyComponentFactory method stageProcessorProvider.

private Map.Entry<String, Provider<StreamlineProcessor>> stageProcessorProvider() {
    Provider<StreamlineProcessor> provider = new Provider<StreamlineProcessor>() {

        @Override
        public StreamlineProcessor create(TopologyComponent component) {
            Object stageConfig = component.getConfig().getAny(StageProcessor.CONFIG_KEY_STAGE);
            ObjectMapper objectMapper = new ObjectMapper();
            StageAction stageAction = objectMapper.convertValue(stageConfig, StageAction.class);
            StageProcessor stageProcessor = new StageProcessor();
            stageProcessor.setStageAction(stageAction);
            return stageProcessor;
        }
    };
    return new SimpleImmutableEntry<>(STAGE, provider);
}

Example 15 with StreamlineProcessor

use of com.hortonworks.streamline.streams.layout.component.StreamlineProcessor in project streamline by hortonworks.

the class TopologyComponentFactory method splitProcessorProvider.

private Map.Entry<String, Provider<StreamlineProcessor>> splitProcessorProvider() {
    Provider<StreamlineProcessor> provider = new Provider<StreamlineProcessor>() {

        @Override
        public StreamlineProcessor create(TopologyComponent component) {
            Object splitConfig = component.getConfig().getAny(SplitProcessor.CONFIG_KEY_SPLIT);
            ObjectMapper objectMapper = new ObjectMapper();
            SplitAction splitAction = objectMapper.convertValue(splitConfig, SplitAction.class);
            SplitProcessor splitProcessor = new SplitProcessor();
            if (component instanceof TopologyOutputComponent) {
                splitProcessor.addOutputStreams(createOutputStreams((TopologyOutputComponent) component));
            } else {
                throw new IllegalArgumentException("Component " + component + " must be an instance of TopologyOutputComponent");
            }
            splitProcessor.setSplitAction(splitAction);
            return splitProcessor;
        }
    };
    return new SimpleImmutableEntry<>(SPLIT, provider);
}