Examples with TopologyOutputComponent com.hortonworks.streamline.streams.catalog.TopologyOutputComponent used on opensource projects

Example 1 with TopologyOutputComponent

use of com.hortonworks.streamline.streams.catalog.TopologyOutputComponent in project streamline by hortonworks.

the class StreamCatalogService method validateEdge.

// validate from, to and stream ids of the edge
private void validateEdge(TopologyEdge edge) {
    TopologyOutputComponent from = getFrom(edge);
    if ((from == null || !from.getTopologyId().equals(edge.getTopologyId()))) {
        throw new IllegalArgumentException("Invalid source for edge " + edge);
    }
    TopologyComponent to = getTo(edge);
    if ((to == null || !to.getTopologyId().equals(edge.getTopologyId()))) {
        throw new IllegalArgumentException("Invalid destination for edge " + edge);
    }
    Set<Long> outputStreamIds = new HashSet<>();
    if (from.getOutputStreamIds() != null) {
        outputStreamIds.addAll(from.getOutputStreamIds());
    } else if (from.getOutputStreams() != null) {
        outputStreamIds.addAll(Collections2.transform(from.getOutputStreams(), new Function<TopologyStream, Long>() {

            @Override
            public Long apply(TopologyStream input) {
                return input.getId();
            }
        }));
    }
    Collection<Long> edgeStreamIds = Collections2.transform(edge.getStreamGroupings(), new Function<StreamGrouping, Long>() {

        public Long apply(StreamGrouping streamGrouping) {
            return streamGrouping.getStreamId();
        }
    });
    if (!outputStreamIds.containsAll(edgeStreamIds)) {
        throw new IllegalArgumentException("Edge stream Ids " + edgeStreamIds + " must be a subset of outputStreamIds " + outputStreamIds);
    }
    // check the fields specified in the fields grouping is a subset of the stream fields
    for (StreamGrouping streamGrouping : edge.getStreamGroupings()) {
        List<String> fields;
        if ((fields = streamGrouping.getFields()) != null) {
            Set<String> schemaFieldPatterns = getFieldPatterns(getStreamInfo(edge.getTopologyId(), streamGrouping.getStreamId(), edge.getVersionId()).getFields());
            fields.forEach(field -> {
                schemaFieldPatterns.stream().filter(pat -> field.matches(pat)).findAny().orElseThrow(() -> new IllegalArgumentException("Fields in the grouping " + fields + " must be a subset the stream fields " + schemaFieldPatterns));
            });
        }
    }
}

Example 2 with TopologyOutputComponent

use of com.hortonworks.streamline.streams.catalog.TopologyOutputComponent in project streamline by hortonworks.

the class TopologyComponentFactory method createRulesProcessorProvider.

private Provider<StreamlineProcessor> createRulesProcessorProvider(final RuleExtractor ruleExtractor) {
    return new Provider<StreamlineProcessor>() {

        @Override
        public StreamlineProcessor create(TopologyComponent component) {
            RulesProcessor processor = new RulesProcessor();
            ObjectMapper objectMapper = new ObjectMapper();
            if (component instanceof TopologyOutputComponent) {
                Set<Stream> outputStreams = createOutputStreams((TopologyOutputComponent) component);
                processor.addOutputStreams(outputStreams);
            } else {
                throw new IllegalArgumentException("Component " + component + " must be an instance of TopologyOutputComponent");
            }
            boolean processAll = component.getConfig().getBoolean(RulesProcessor.CONFIG_PROCESS_ALL, true);
            processor.setProcessAll(processAll);
            Object ruleList = component.getConfig().getAny(RulesProcessor.CONFIG_KEY_RULES);
            List<Long> ruleIds = objectMapper.convertValue(ruleList, new TypeReference<List<Long>>() {
            });
            try {
                List<Rule> rules = new ArrayList<>();
                for (Long ruleId : ruleIds) {
                    rules.add(ruleExtractor.getRule(component.getTopologyId(), ruleId, component.getVersionId()));
                }
                processor.setRules(rules);
            } catch (Exception ex) {
                throw new RuntimeException(ex);
            }
            return processor;
        }
    };
}

Example 3 with TopologyOutputComponent

use of com.hortonworks.streamline.streams.catalog.TopologyOutputComponent in project streamline by hortonworks.

the class TopologyComponentFactory method normalizationProcessorProvider.

private Map.Entry<String, Provider<StreamlineProcessor>> normalizationProcessorProvider() {
    Provider<StreamlineProcessor> provider = new Provider<StreamlineProcessor>() {

        @Override
        public StreamlineProcessor create(TopologyComponent component) {
            Config config = component.getConfig();
            Object typeObj = config.getAny(NormalizationProcessor.CONFIG_KEY_TYPE);
            Object normConfObj = config.getAny(NormalizationProcessor.CONFIG_KEY_NORMALIZATION);
            ObjectMapper objectMapper = new ObjectMapper();
            NormalizationProcessor.Type type = objectMapper.convertValue(typeObj, NormalizationProcessor.Type.class);
            Map<String, NormalizationConfig> normConfig = objectMapper.convertValue(normConfObj, new TypeReference<Map<String, NormalizationConfig>>() {
            });
            updateWithSchemas(component.getTopologyId(), component.getVersionId(), normConfig);
            Set<Stream> outputStreams;
            if (component instanceof TopologyOutputComponent) {
                outputStreams = createOutputStreams((TopologyOutputComponent) component);
            } else {
                throw new IllegalArgumentException("Component " + component + " must be an instance of TopologyOutputComponent");
            }
            if (outputStreams.size() != 1) {
                throw new IllegalArgumentException("Normalization component [" + component + "] must have only one output stream");
            }
            return new NormalizationProcessor(normConfig, outputStreams.iterator().next(), type);
        }
    };
    return new SimpleImmutableEntry<>(NORMALIZATION, provider);
}

Example 4 with TopologyOutputComponent

use of com.hortonworks.streamline.streams.catalog.TopologyOutputComponent in project streamline by hortonworks.

the class TopologyComponentFactory method splitProcessorProvider.

private Map.Entry<String, Provider<StreamlineProcessor>> splitProcessorProvider() {
    Provider<StreamlineProcessor> provider = new Provider<StreamlineProcessor>() {

        @Override
        public StreamlineProcessor create(TopologyComponent component) {
            Object splitConfig = component.getConfig().getAny(SplitProcessor.CONFIG_KEY_SPLIT);
            ObjectMapper objectMapper = new ObjectMapper();
            SplitAction splitAction = objectMapper.convertValue(splitConfig, SplitAction.class);
            SplitProcessor splitProcessor = new SplitProcessor();
            if (component instanceof TopologyOutputComponent) {
                splitProcessor.addOutputStreams(createOutputStreams((TopologyOutputComponent) component));
            } else {
                throw new IllegalArgumentException("Component " + component + " must be an instance of TopologyOutputComponent");
            }
            splitProcessor.setSplitAction(splitAction);
            return splitProcessor;
        }
    };
    return new SimpleImmutableEntry<>(SPLIT, provider);
}