Examples with Rule com.hortonworks.streamline.streams.layout.component.rule.Rule used on opensource projects

Example 11 with Rule

use of com.hortonworks.streamline.streams.layout.component.rule.Rule in project streamline by hortonworks.

the class TopologyExportVisitor method createRuleHandlerMap.

private Map<String, RulesHandler> createRuleHandlerMap() {
    ImmutableMap.Builder<String, RulesHandler> rulesHandlerBuilder = ImmutableMap.builder();
    RulesHandler rulesHandler = new RulesHandler() {

        @Override
        public void handle(Rule rule) throws Exception {
            TopologyRule topologyRule = streamCatalogService.getRule(topologyId, rule.getId());
            topologyData.addRule(topologyRule);
        }
    };
    rulesHandlerBuilder.put(ComponentTypes.RULE, rulesHandler);
    rulesHandlerBuilder.put(ComponentTypes.PROJECTION, rulesHandler);
    rulesHandlerBuilder.put(ComponentTypes.BRANCH, new RulesHandler() {

        @Override
        public void handle(Rule rule) throws Exception {
            TopologyBranchRule topologyBranchRule = streamCatalogService.getBranchRule(topologyId, rule.getId());
            topologyData.addBranch(topologyBranchRule);
        }
    });
    rulesHandlerBuilder.put(ComponentTypes.WINDOW, new RulesHandler() {

        @Override
        public void handle(Rule rule) throws Exception {
            TopologyWindow topologyWindow = streamCatalogService.getWindow(topologyId, rule.getId());
            topologyData.addWindow(topologyWindow);
        }
    });
    return rulesHandlerBuilder.build();
}

Example 12 with Rule

use of com.hortonworks.streamline.streams.layout.component.rule.Rule in project streamline by hortonworks.

the class StreamCatalogService method parseAndSerialize.

private String parseAndSerialize(TopologyRule topologyRule) throws JsonProcessingException {
    Rule rule = new Rule();
    rule.setId(topologyRule.getId());
    rule.setName(topologyRule.getName());
    rule.setDescription(topologyRule.getDescription());
    rule.setWindow(topologyRule.getWindow());
    rule.setActions(topologyRule.getActions());
    if (topologyRule.getStreams() != null && !topologyRule.getStreams().isEmpty()) {
        topologyRule.setSql(getSqlString(topologyRule.getStreams(), topologyRule.getProjections(), topologyRule.getCondition(), null));
    } else if (StringUtils.isEmpty(topologyRule.getSql())) {
        throw new IllegalArgumentException("Either streams or sql string should be specified.");
    }
    updateRuleWithSql(rule, topologyRule.getSql(), topologyRule.getTopologyId(), topologyRule.getVersionId());
    ObjectMapper mapper = new ObjectMapper();
    return mapper.writeValueAsString(rule);
}

Example 13 with Rule

use of com.hortonworks.streamline.streams.layout.component.rule.Rule in project streamline by hortonworks.

the class StormTopologyFluxGenerator method handleWindowedRules.

private void handleWindowedRules(RulesProcessor rulesProcessor, List<Rule> rulesWithWindow) {
    // assert that RulesProcessor only has a windowed rule, not multiple rules.
    if (rulesWithWindow.size() > 1) {
        throw new IllegalStateException("Windowed RulesProcessor should have only one rule.");
    }
    Rule rule = rulesWithWindow.get(0);
    Collection<Rule> rules = Collections.singletonList(rule);
    Window window = rulesWithWindow.get(0).getWindow();
    // create windowed bolt per unique window configuration
    RulesProcessor windowedRulesProcessor = copyRulesProcessor(rulesProcessor);
    windowedRulesProcessor.setRules(new ArrayList<>(rules));
    windowedRulesProcessor.setId(rulesProcessor.getId());
    windowedRulesProcessor.setName(rulesProcessor.getName());
    windowedRulesProcessor.getConfig().setAny(RulesProcessor.CONFIG_KEY_RULES, Collections2.transform(rules, new Function<Rule, Long>() {

        @Override
        public Long apply(Rule input) {
            return input.getId();
        }
    }));
    LOG.debug("Rules processor with window {}", windowedRulesProcessor);
    keysAndComponents.add(makeEntry(StormTopologyLayoutConstants.YAML_KEY_BOLTS, getYamlComponents(fluxComponentFactory.getFluxComponent(windowedRulesProcessor), windowedRulesProcessor)));
    List<Edge> originEdgesTo = topologyDag.getEdgesTo(rulesProcessor);
    List<Edge> originEdgesFrom = topologyDag.getEdgesFrom(rulesProcessor);
    // remove streams before wiring
    removeFluxStreamsTo(getFluxId(rulesProcessor));
    removeFluxStreamsFrom(getFluxId(rulesProcessor));
    // Wire the windowed bolt with the appropriate edges
    wireWindowedRulesProcessor(windowedRulesProcessor, originEdgesTo, originEdgesFrom);
    mayBeUpdateTopologyConfig(window);
    edgeAlreadyAddedComponents.add(getFluxId(rulesProcessor));
}

Example 14 with Rule

use of com.hortonworks.streamline.streams.layout.component.rule.Rule in project streamline by hortonworks.

the class StormTopologyDependenciesHandler method visit.

@Override
public void visit(RulesProcessor rulesProcessor) {
    Set<UDF> udfsToShip = new HashSet<>();
    for (Rule rule : rulesProcessor.getRules()) {
        for (Udf udf : rule.getReferredUdfs()) {
            List<QueryParam> qps = QueryParam.params(UDF.NAME, udf.getName());
            // The null check for backward compatibility
            if (udf.getClassName() != null) {
                qps.add(new QueryParam(UDF.CLASSNAME, udf.getClassName()));
            }
            // The null check for backward compatibility
            if (udf.getType() != null) {
                qps.add(new QueryParam(UDF.TYPE, udf.getType().toString()));
            }
            Collection<UDF> udfs = catalogService.listUDFs(qps);
            if (udfs.size() > 1) {
                throw new IllegalStateException("Multiple UDF definitions for :" + udf);
            } else if (udfs.size() == 1) {
                udfsToShip.add(udfs.iterator().next());
            } else {
                throw new IllegalStateException("No catalog entity for udf: '" + udf + "'. " + "May be the UDF information is not bootstrapped or got deleted.");
            }
        }
        for (UDF udf : udfsToShip) {
            extraJars.add(udf.getJarStoragePath());
        }
    }
    resourceNames.addAll(rulesProcessor.getExtraResources());
    handleBundleForStreamlineComponent(rulesProcessor);
}

Example 15 with Rule

use of com.hortonworks.streamline.streams.layout.component.rule.Rule in project streamline by hortonworks.

the class StreamCatalogService method doImportTopology.

private Topology doImportTopology(Topology newTopology, TopologyData topologyData) throws Exception {
    List<TopologySource> topologySources = topologyData.getSources();
    Map<Long, Long> oldToNewComponentIds = new HashMap<>();
    Map<Long, Long> oldToNewRuleIds = new HashMap<>();
    Map<Long, Long> oldToNewWindowIds = new HashMap<>();
    Map<Long, Long> oldToNewBranchRuleIds = new HashMap<>();
    Map<Long, Long> oldToNewStreamIds = new HashMap<>();
    // import source streams
    for (TopologySource topologySource : topologySources) {
        topologySource.setOutputStreamIds(importOutputStreams(newTopology.getId(), oldToNewStreamIds, topologySource.getOutputStreams()));
        topologySource.setOutputStreams(null);
    }
    // import processor streams
    for (TopologyProcessor topologyProcessor : topologyData.getProcessors()) {
        topologyProcessor.setOutputStreamIds(importOutputStreams(newTopology.getId(), oldToNewStreamIds, topologyProcessor.getOutputStreams()));
        topologyProcessor.setOutputStreams(null);
    }
    // import rules
    for (TopologyRule rule : topologyData.getRules()) {
        Long currentId = rule.getId();
        rule.setId(null);
        TopologyRule addedRule = addRule(newTopology.getId(), rule);
        oldToNewRuleIds.put(currentId, addedRule.getId());
    }
    // import windowed rules
    for (TopologyWindow window : topologyData.getWindows()) {
        Long currentId = window.getId();
        window.setId(null);
        TopologyWindow addedWindow = addWindow(newTopology.getId(), window);
        oldToNewWindowIds.put(currentId, addedWindow.getId());
    }
    // import branch rules
    for (TopologyBranchRule branchRule : topologyData.getBranchRules()) {
        Long currentId = branchRule.getId();
        branchRule.setId(null);
        TopologyBranchRule addedBranchRule = addBranchRule(newTopology.getId(), branchRule);
        oldToNewBranchRuleIds.put(currentId, addedBranchRule.getId());
    }
    // import sources
    for (TopologySource topologySource : topologySources) {
        Long oldComponentId = topologySource.getId();
        topologySource.setId(null);
        topologySource.setTopologyId(newTopology.getId());
        TopologyComponentBundle bundle = getCurrentTopologyComponentBundle(TopologyComponentBundle.TopologyComponentType.SOURCE, topologyData.getBundleIdToType().get(topologySource.getTopologyComponentBundleId().toString()));
        topologySource.setTopologyComponentBundleId(bundle.getId());
        addTopologySource(newTopology.getId(), topologySource);
        oldToNewComponentIds.put(oldComponentId, topologySource.getId());
    }
    // import processors
    for (TopologyProcessor topologyProcessor : topologyData.getProcessors()) {
        Long oldComponentId = topologyProcessor.getId();
        topologyProcessor.setId(null);
        topologyProcessor.setTopologyId(newTopology.getId());
        TopologyComponentBundle bundle;
        String subType = topologyData.getBundleIdToType().get(topologyProcessor.getTopologyComponentBundleId().toString());
        if (TopologyLayoutConstants.JSON_KEY_CUSTOM_PROCESSOR_SUB_TYPE.equals(subType)) {
            QueryParam queryParam = new QueryParam(CustomProcessorInfo.NAME, topologyProcessor.getConfig().get(CustomProcessorInfo.NAME));
            Collection<TopologyComponentBundle> result = listCustomProcessorBundlesWithFilter(Collections.singletonList(queryParam));
            if (result.size() != 1) {
                throw new IllegalStateException("Not able to find topology component bundle for custom processor :" + topologyProcessor.getConfig().get(CustomProcessorInfo.NAME));
            }
            bundle = result.iterator().next();
        } else {
            bundle = getCurrentTopologyComponentBundle(TopologyComponentBundle.TopologyComponentType.PROCESSOR, subType);
        }
        topologyProcessor.setTopologyComponentBundleId(bundle.getId());
        Optional<Object> ruleListObj = topologyProcessor.getConfig().getAnyOptional(RulesProcessor.CONFIG_KEY_RULES);
        ruleListObj.ifPresent(ruleList -> {
            List<Long> ruleIds = new ObjectMapper().convertValue(ruleList, new TypeReference<List<Long>>() {
            });
            List<Long> updatedRuleIds = new ArrayList<>();
            if (ComponentTypes.RULE.equals(bundle.getSubType()) || ComponentTypes.PROJECTION.equals(bundle.getSubType())) {
                ruleIds.forEach(ruleId -> updatedRuleIds.add(oldToNewRuleIds.get(ruleId)));
            } else if (bundle.getSubType().equals(ComponentTypes.BRANCH)) {
                ruleIds.forEach(ruleId -> updatedRuleIds.add(oldToNewBranchRuleIds.get(ruleId)));
            } else if (bundle.getSubType().equals(ComponentTypes.WINDOW)) {
                ruleIds.forEach(ruleId -> updatedRuleIds.add(oldToNewWindowIds.get(ruleId)));
            }
            topologyProcessor.getConfig().setAny(RulesProcessor.CONFIG_KEY_RULES, updatedRuleIds);
        });
        addTopologyProcessor(newTopology.getId(), topologyProcessor);
        oldToNewComponentIds.put(oldComponentId, topologyProcessor.getId());
    }
    // import sinks
    for (TopologySink topologySink : topologyData.getSinks()) {
        topologySink.setTopologyId(newTopology.getId());
        Long currentId = topologySink.getId();
        topologySink.setId(null);
        TopologyComponentBundle bundle = getCurrentTopologyComponentBundle(TopologyComponentBundle.TopologyComponentType.SINK, topologyData.getBundleIdToType().get(topologySink.getTopologyComponentBundleId().toString()));
        topologySink.setTopologyComponentBundleId(bundle.getId());
        if (bundle.getSubType().equals(NOTIFICATION)) {
            updateNotifierJarFileName(topologySink);
        }
        addTopologySink(newTopology.getId(), topologySink);
        oldToNewComponentIds.put(currentId, topologySink.getId());
    }
    // import edges
    for (TopologyEdge topologyEdge : topologyData.getEdges()) {
        List<StreamGrouping> streamGroupings = topologyEdge.getStreamGroupings();
        for (StreamGrouping streamGrouping : streamGroupings) {
            Long newStreamId = oldToNewStreamIds.get(streamGrouping.getStreamId());
            streamGrouping.setStreamId(newStreamId);
        }
        topologyEdge.setId(null);
        topologyEdge.setTopologyId(newTopology.getId());
        topologyEdge.setFromId(oldToNewComponentIds.get(topologyEdge.getFromId()));
        topologyEdge.setToId(oldToNewComponentIds.get(topologyEdge.getToId()));
        addTopologyEdge(newTopology.getId(), topologyEdge);
    }
    // import topology editor metadata
    TopologyEditorMetadata topologyEditorMetadata = topologyData.getTopologyEditorMetadata();
    topologyEditorMetadata.setTopologyId(newTopology.getId());
    if (topologyEditorMetadata.getData() != null) {
        TopologyUIData topologyUIData = new ObjectMapper().readValue(topologyEditorMetadata.getData(), TopologyUIData.class);
        topologyUIData.getSources().forEach(c -> c.setId(oldToNewComponentIds.get(c.getId())));
        topologyUIData.getProcessors().forEach(c -> c.setId(oldToNewComponentIds.get(c.getId())));
        topologyUIData.getSinks().forEach(c -> c.setId(oldToNewComponentIds.get(c.getId())));
        topologyEditorMetadata.setData(new ObjectMapper().writeValueAsString(topologyUIData));
    } else {
        topologyEditorMetadata.setData(StringUtils.EMPTY);
    }
    addTopologyEditorMetadata(newTopology.getId(), topologyData.getTopologyEditorMetadata());
    return newTopology;
}