Examples with StreamlineSource com.hortonworks.streamline.streams.layout.component.StreamlineSource used on opensource projects

Example 1 with StreamlineSource

use of com.hortonworks.streamline.streams.layout.component.StreamlineSource in project streamline by hortonworks.

the class FileReaderSpoutFluxComponent method generateComponent.

@Override
protected void generateComponent() {
    StreamlineSource streamlineSource = (StreamlineSource) conf.get(StormTopologyLayoutConstants.STREAMLINE_COMPONENT_CONF_KEY);
    // add the output stream to conf so that the kafka spout declares output stream properly
    if (streamlineSource != null && streamlineSource.getOutputStreams().size() == 1) {
        conf.put(TopologyLayoutConstants.JSON_KEY_OUTPUT_STREAM_ID, streamlineSource.getOutputStreams().iterator().next().getId());
    } else {
        String msg = "FileReaderSpout source component [" + streamlineSource + "] should define exactly one output stream for Storm";
        LOG.error(msg, streamlineSource);
        throw new IllegalArgumentException(msg);
    }
    String spoutId = "fileReaderSpout" + UUID_FOR_COMPONENTS;
    String spoutClassName = "com.hortonworks.streamline.examples.sources.FileReaderSpout";
    String[] contructorArgNames = { "path" };
    List<Object> configMethods = new ArrayList<>();
    String[] configMethodNames = { "withOutputStream", "withDelimiter" };
    String[] configKeys = { TopologyLayoutConstants.JSON_KEY_OUTPUT_STREAM_ID, "delimiter" };
    configMethods.addAll(getConfigMethodsYaml(configMethodNames, configKeys));
    component = createComponent(spoutId, spoutClassName, null, getConstructorArgsYaml(contructorArgNames), configMethods);
    addParallelismToComponent();
}

Example 2 with StreamlineSource

use of com.hortonworks.streamline.streams.layout.component.StreamlineSource in project streamline by hortonworks.

the class TestTopologyDagCreatingVisitorTest method visitProcessor_connectedFromSource.

@Test
public void visitProcessor_connectedFromSource() throws Exception {
    StreamlineSource originSource = TopologyTestHelper.createStreamlineSource("1");
    StreamlineProcessor originProcessor = TopologyTestHelper.createStreamlineProcessor("2");
    TopologyDag originTopologyDag = new TopologyDag();
    originTopologyDag.add(originSource);
    originTopologyDag.add(originProcessor);
    originTopologyDag.addEdge(new Edge("e1", originSource, originProcessor, "default", Stream.Grouping.SHUFFLE));
    TestRunSource testSource = createTestRunSource(originSource);
    TestRunProcessor testProcessor = createTestRunProcessor(originProcessor);
    TestTopologyDagCreatingVisitor visitor = new TestTopologyDagCreatingVisitor(originTopologyDag, Collections.singletonMap(originSource.getName(), testSource), Collections.singletonMap(originProcessor.getName(), testProcessor), Collections.emptyMap(), Collections.emptyMap());
    visitor.visit(originSource);
    visitor.visit(originProcessor);
    TopologyDag testTopologyDag = visitor.getTestTopologyDag();
    List<OutputComponent> testProcessors = testTopologyDag.getOutputComponents().stream().filter(o -> (o instanceof TestRunProcessor && o.getName().equals(originProcessor.getName()))).collect(toList());
    List<OutputComponent> testSources = testTopologyDag.getOutputComponents().stream().filter(o -> (o instanceof TestRunSource && o.getName().equals(originSource.getName()))).collect(toList());
    TestRunProcessor testRunProcessor = (TestRunProcessor) testProcessors.get(0);
    TestRunSource testRunSource = (TestRunSource) testSources.get(0);
    assertEquals(1, testTopologyDag.getEdgesFrom(testRunSource).size());
    assertEquals(1, testTopologyDag.getEdgesTo(testRunProcessor).size());
    assertTrue(testTopologyDag.getEdgesFrom(testRunSource).get(0) == testTopologyDag.getEdgesTo(testRunProcessor).get(0));
}

Example 3 with StreamlineSource

use of com.hortonworks.streamline.streams.layout.component.StreamlineSource in project streamline by hortonworks.

the class TestTopologyDagCreatingVisitorTest method visitSource_noMatchingTestRunSource.

@Test
public void visitSource_noMatchingTestRunSource() throws Exception {
    StreamlineSource originSource = TopologyTestHelper.createStreamlineSource("1");
    TopologyDag originTopologyDag = new TopologyDag();
    originTopologyDag.add(originSource);
    TestTopologyDagCreatingVisitor visitor = new TestTopologyDagCreatingVisitor(originTopologyDag, Collections.emptyMap(), Collections.emptyMap(), Collections.emptyMap(), Collections.emptyMap());
    try {
        visitor.visit(originSource);
        fail("IllegalStateException should be thrown.");
    } catch (IllegalStateException e) {
        assertTrue(e.getMessage().contains(originSource.getName()));
    }
}

Example 4 with StreamlineSource

use of com.hortonworks.streamline.streams.layout.component.StreamlineSource in project streamline by hortonworks.

the class CatalogToLayoutConverter method getComponentLayout.

public static com.hortonworks.streamline.streams.layout.component.Component getComponentLayout(TopologyComponent component) {
    StreamlineComponent componentLayout;
    if (component instanceof TopologySource) {
        componentLayout = new StreamlineSource() {

            @Override
            public void accept(TopologyDagVisitor visitor) {
                throw new UnsupportedOperationException("Not intended to be called here.");
            }
        };
    } else if (component instanceof TopologyProcessor) {
        componentLayout = new StreamlineProcessor() {

            @Override
            public void accept(TopologyDagVisitor visitor) {
                throw new UnsupportedOperationException("Not intended to be called here.");
            }
        };
    } else if (component instanceof TopologySink) {
        componentLayout = new StreamlineSink() {

            @Override
            public void accept(TopologyDagVisitor visitor) {
                throw new UnsupportedOperationException("Not intended to be called here.");
            }
        };
    } else {
        componentLayout = new StreamlineComponent() {

            @Override
            public void accept(TopologyDagVisitor visitor) {
                throw new UnsupportedOperationException("Not intended to be called here.");
            }
        };
    }
    componentLayout.setId(component.getId().toString());
    componentLayout.setName(component.getName());
    componentLayout.setConfig(component.getConfig());
    return componentLayout;
}

Example 5 with StreamlineSource

use of com.hortonworks.streamline.streams.layout.component.StreamlineSource in project streamline by hortonworks.

the class StormTopologyTimeSeriesMetricsImplTest method getSourceLayoutForTest.

private Component getSourceLayoutForTest() {
    StreamlineSource source = new StreamlineSource() {

        @Override
        public void accept(TopologyDagVisitor visitor) {
        }
    };
    source.setId("11");
    source.setName("device");
    Map<String, String> configurations = buildSourceConfigWithKafkaTopic(TOPIC_NAME);
    source.setConfig(new Config(configurations));
    return source;
}