Examples with StreamlineProcessor com.hortonworks.streamline.streams.layout.component.StreamlineProcessor used on opensource projects

Example 1 with StreamlineProcessor

use of com.hortonworks.streamline.streams.layout.component.StreamlineProcessor in project streamline by hortonworks.

the class TestTopologyDagCreatingVisitorTest method visitProcessor_connectedFromSource.

@Test
public void visitProcessor_connectedFromSource() throws Exception {
    StreamlineSource originSource = TopologyTestHelper.createStreamlineSource("1");
    StreamlineProcessor originProcessor = TopologyTestHelper.createStreamlineProcessor("2");
    TopologyDag originTopologyDag = new TopologyDag();
    originTopologyDag.add(originSource);
    originTopologyDag.add(originProcessor);
    originTopologyDag.addEdge(new Edge("e1", originSource, originProcessor, "default", Stream.Grouping.SHUFFLE));
    TestRunSource testSource = createTestRunSource(originSource);
    TestRunProcessor testProcessor = createTestRunProcessor(originProcessor);
    TestTopologyDagCreatingVisitor visitor = new TestTopologyDagCreatingVisitor(originTopologyDag, Collections.singletonMap(originSource.getName(), testSource), Collections.singletonMap(originProcessor.getName(), testProcessor), Collections.emptyMap(), Collections.emptyMap());
    visitor.visit(originSource);
    visitor.visit(originProcessor);
    TopologyDag testTopologyDag = visitor.getTestTopologyDag();
    List<OutputComponent> testProcessors = testTopologyDag.getOutputComponents().stream().filter(o -> (o instanceof TestRunProcessor && o.getName().equals(originProcessor.getName()))).collect(toList());
    List<OutputComponent> testSources = testTopologyDag.getOutputComponents().stream().filter(o -> (o instanceof TestRunSource && o.getName().equals(originSource.getName()))).collect(toList());
    TestRunProcessor testRunProcessor = (TestRunProcessor) testProcessors.get(0);
    TestRunSource testRunSource = (TestRunSource) testSources.get(0);
    assertEquals(1, testTopologyDag.getEdgesFrom(testRunSource).size());
    assertEquals(1, testTopologyDag.getEdgesTo(testRunProcessor).size());
    assertTrue(testTopologyDag.getEdgesFrom(testRunSource).get(0) == testTopologyDag.getEdgesTo(testRunProcessor).get(0));
}

Example 2 with StreamlineProcessor

use of com.hortonworks.streamline.streams.layout.component.StreamlineProcessor in project streamline by hortonworks.

the class TopologyTestHelper method createStreamlineProcessor.

public static StreamlineProcessor createStreamlineProcessor(String id) {
    Stream stream = createDefaultStream();
    StreamlineProcessor processor = new StreamlineProcessor(Sets.newHashSet(stream));
    processor.setId(id);
    processor.setName("testProcessor_" + id);
    processor.setConfig(new Config());
    processor.setTransformationClass("dummyTransformation");
    return processor;
}

Example 3 with StreamlineProcessor

use of com.hortonworks.streamline.streams.layout.component.StreamlineProcessor in project streamline by hortonworks.

the class TopologyComponentFactory method modelProcessorProvider.

private Map.Entry<String, Provider<StreamlineProcessor>> modelProcessorProvider() {
    Provider<StreamlineProcessor> provider = new Provider<StreamlineProcessor>() {

        @Override
        public StreamlineProcessor create(TopologyComponent component) {
            String modelName = component.getConfig().getString(ModelProcessor.CONFIG_MODEL_NAME, StringUtils.EMPTY);
            ModelProcessor modelProcessor = new ModelProcessor();
            if (!modelName.equals(StringUtils.EMPTY)) {
                modelProcessor.setPmml(modelRegistryClient.getMLModelContents(modelName));
            }
            return modelProcessor;
        }
    };
    return new SimpleImmutableEntry<>(PMML, provider);
}

Example 4 with StreamlineProcessor

use of com.hortonworks.streamline.streams.layout.component.StreamlineProcessor in project streamline by hortonworks.

the class TopologyComponentFactory method normalizationProcessorProvider.

private Map.Entry<String, Provider<StreamlineProcessor>> normalizationProcessorProvider() {
    Provider<StreamlineProcessor> provider = new Provider<StreamlineProcessor>() {

        @Override
        public StreamlineProcessor create(TopologyComponent component) {
            Config config = component.getConfig();
            Object typeObj = config.getAny(NormalizationProcessor.CONFIG_KEY_TYPE);
            Object normConfObj = config.getAny(NormalizationProcessor.CONFIG_KEY_NORMALIZATION);
            ObjectMapper objectMapper = new ObjectMapper();
            NormalizationProcessor.Type type = objectMapper.convertValue(typeObj, NormalizationProcessor.Type.class);
            Map<String, NormalizationConfig> normConfig = objectMapper.convertValue(normConfObj, new TypeReference<Map<String, NormalizationConfig>>() {
            });
            updateWithSchemas(component.getTopologyId(), component.getVersionId(), normConfig);
            Set<Stream> outputStreams;
            if (component instanceof TopologyOutputComponent) {
                outputStreams = createOutputStreams((TopologyOutputComponent) component);
            } else {
                throw new IllegalArgumentException("Component " + component + " must be an instance of TopologyOutputComponent");
            }
            if (outputStreams.size() != 1) {
                throw new IllegalArgumentException("Normalization component [" + component + "] must have only one output stream");
            }
            return new NormalizationProcessor(normConfig, outputStreams.iterator().next(), type);
        }
    };
    return new SimpleImmutableEntry<>(NORMALIZATION, provider);
}

Example 5 with StreamlineProcessor

use of com.hortonworks.streamline.streams.layout.component.StreamlineProcessor in project streamline by hortonworks.

the class CatalogToLayoutConverter method getComponentLayout.

public static com.hortonworks.streamline.streams.layout.component.Component getComponentLayout(TopologyComponent component) {
    StreamlineComponent componentLayout;
    if (component instanceof TopologySource) {
        componentLayout = new StreamlineSource() {

            @Override
            public void accept(TopologyDagVisitor visitor) {
                throw new UnsupportedOperationException("Not intended to be called here.");
            }
        };
    } else if (component instanceof TopologyProcessor) {
        componentLayout = new StreamlineProcessor() {

            @Override
            public void accept(TopologyDagVisitor visitor) {
                throw new UnsupportedOperationException("Not intended to be called here.");
            }
        };
    } else if (component instanceof TopologySink) {
        componentLayout = new StreamlineSink() {

            @Override
            public void accept(TopologyDagVisitor visitor) {
                throw new UnsupportedOperationException("Not intended to be called here.");
            }
        };
    } else {
        componentLayout = new StreamlineComponent() {

            @Override
            public void accept(TopologyDagVisitor visitor) {
                throw new UnsupportedOperationException("Not intended to be called here.");
            }
        };
    }
    componentLayout.setId(component.getId().toString());
    componentLayout.setName(component.getName());
    componentLayout.setConfig(component.getConfig());
    return componentLayout;
}