Examples with Stream com.hortonworks.streamline.streams.layout.component.Stream used on opensource projects

Example 1 with Stream

use of com.hortonworks.streamline.streams.layout.component.Stream in project streamline by hortonworks.

the class TestTopologyDagCreatingVisitorTest method createTestRunSource.

private TestRunSource createTestRunSource(StreamlineSource originSource) {
    Map<String, List<Map<String, Object>>> testRecordsMap = new HashMap<>();
    for (Stream stream : originSource.getOutputStreams()) {
        testRecordsMap.put(stream.getId(), TopologyTestHelper.createTestRecords());
    }
    TestRunSource testRunSource = new TestRunSource(originSource.getOutputStreams(), testRecordsMap, 1, 0L, "");
    testRunSource.setName(originSource.getName());
    return testRunSource;
}

Example 2 with Stream

use of com.hortonworks.streamline.streams.layout.component.Stream in project streamline by hortonworks.

the class TestTopologyDagCreatingVisitorTest method visitProcessor_connectedFromSource.

@Test
public void visitProcessor_connectedFromSource() throws Exception {
    StreamlineSource originSource = TopologyTestHelper.createStreamlineSource("1");
    StreamlineProcessor originProcessor = TopologyTestHelper.createStreamlineProcessor("2");
    TopologyDag originTopologyDag = new TopologyDag();
    originTopologyDag.add(originSource);
    originTopologyDag.add(originProcessor);
    originTopologyDag.addEdge(new Edge("e1", originSource, originProcessor, "default", Stream.Grouping.SHUFFLE));
    TestRunSource testSource = createTestRunSource(originSource);
    TestRunProcessor testProcessor = createTestRunProcessor(originProcessor);
    TestTopologyDagCreatingVisitor visitor = new TestTopologyDagCreatingVisitor(originTopologyDag, Collections.singletonMap(originSource.getName(), testSource), Collections.singletonMap(originProcessor.getName(), testProcessor), Collections.emptyMap(), Collections.emptyMap());
    visitor.visit(originSource);
    visitor.visit(originProcessor);
    TopologyDag testTopologyDag = visitor.getTestTopologyDag();
    List<OutputComponent> testProcessors = testTopologyDag.getOutputComponents().stream().filter(o -> (o instanceof TestRunProcessor && o.getName().equals(originProcessor.getName()))).collect(toList());
    List<OutputComponent> testSources = testTopologyDag.getOutputComponents().stream().filter(o -> (o instanceof TestRunSource && o.getName().equals(originSource.getName()))).collect(toList());
    TestRunProcessor testRunProcessor = (TestRunProcessor) testProcessors.get(0);
    TestRunSource testRunSource = (TestRunSource) testSources.get(0);
    assertEquals(1, testTopologyDag.getEdgesFrom(testRunSource).size());
    assertEquals(1, testTopologyDag.getEdgesTo(testRunProcessor).size());
    assertTrue(testTopologyDag.getEdgesFrom(testRunSource).get(0) == testTopologyDag.getEdgesTo(testRunProcessor).get(0));
}

Example 3 with Stream

use of com.hortonworks.streamline.streams.layout.component.Stream in project streamline by hortonworks.

the class TopologyTestHelper method createStreamlineProcessor.

public static StreamlineProcessor createStreamlineProcessor(String id) {
    Stream stream = createDefaultStream();
    StreamlineProcessor processor = new StreamlineProcessor(Sets.newHashSet(stream));
    processor.setId(id);
    processor.setName("testProcessor_" + id);
    processor.setConfig(new Config());
    processor.setTransformationClass("dummyTransformation");
    return processor;
}

Example 4 with Stream

use of com.hortonworks.streamline.streams.layout.component.Stream in project streamline by hortonworks.

the class NormalizationBoltTest method createBulkNormalizationProcessor.

public static NormalizationProcessor createBulkNormalizationProcessor(String outputStreamId) throws NormalizationException, IOException {
    Map<String, NormalizationConfig> inputStreamsWithConfig = new HashMap<>();
    Schema inputSchema = Schema.of(Schema.Field.of("temp", Schema.Type.INTEGER), Schema.Field.of("foo", Schema.Type.STRING));
    String bulkScriptText = getBulkScriptText();
    BulkNormalizationConfig bulkNormalizationConfig = new BulkNormalizationConfig(inputSchema, bulkScriptText);
    inputStreamsWithConfig.put(INPUT_STREAM_ID, bulkNormalizationConfig);
    Stream declaredOutputStream = new Stream(outputStreamId, OUTPUT_SCHEMA_FIELDS);
    NormalizationProcessor normalizationProcessor = new NormalizationProcessor(inputStreamsWithConfig, declaredOutputStream, NormalizationProcessor.Type.bulk);
    normalizationProcessor.addOutputStream(declaredOutputStream);
    return normalizationProcessor;
}

Example 5 with Stream

use of com.hortonworks.streamline.streams.layout.component.Stream in project streamline by hortonworks.

the class StreamCatalogService method setReconfigureTarget.

private void setReconfigureTarget(TopologyEdge edge, TopologyStream stream) {
    TopologyComponent component = getTo(edge);
    component.setReconfigure(true);
    dao.addOrUpdate(component);
    // if component is a processor, update any rules in that processor that uses any of the streams
    if (component instanceof TopologyProcessor) {
        setReconfigureRules(Collections.singletonList((TopologyProcessor) component), edge.getStreamGroupings().stream().map(StreamGrouping::getStreamId).map(sid -> getStreamInfo(edge.getTopologyId(), sid, edge.getVersionId())).filter(curStream -> stream == null || curStream.getId().equals(stream.getId())).collect(Collectors.toList()));
    }
}