Examples with TopologyComponentBundle com.hortonworks.streamline.streams.catalog.topology.TopologyComponentBundle used on opensource projects

Example 11 with TopologyComponentBundle

use of com.hortonworks.streamline.streams.catalog.topology.TopologyComponentBundle in project streamline by hortonworks.

the class StreamCatalogService method addOrUpdateTopologyComponentBundle.

public TopologyComponentBundle addOrUpdateTopologyComponentBundle(Long id, TopologyComponentBundle topologyComponentBundle, java.io.File bundleJar) throws ComponentConfigException, IOException {
    topologyComponentBundle.getTopologyComponentUISpecification().validate();
    loadTransformationClassForBundle(topologyComponentBundle, bundleJar);
    if (!topologyComponentBundle.getBuiltin()) {
        topologyComponentBundle.setBundleJar(getTopologyComponentBundleJarName(topologyComponentBundle));
        try (InputStream is = new FileInputStream(bundleJar)) {
            uploadFileToStorage(is, topologyComponentBundle.getBundleJar());
        }
    }
    TopologyComponentBundle existing = new TopologyComponentBundle();
    existing.setId(id);
    existing = this.dao.get(existing.getStorableKey());
    if (!existing.getBuiltin()) {
        try {
            deleteFileFromStorage(existing.getBundleJar());
        } catch (IOException e) {
            if (!topologyComponentBundle.getBuiltin()) {
                deleteFileFromStorage(topologyComponentBundle.getBundleJar());
            }
            throw e;
        }
    }
    try {
        topologyComponentBundle.setId(id);
        topologyComponentBundle.setTimestamp(System.currentTimeMillis());
        this.dao.addOrUpdate(topologyComponentBundle);
    } catch (StorageException e) {
        if (!topologyComponentBundle.getBuiltin()) {
            deleteFileFromStorage(topologyComponentBundle.getBundleJar());
        }
        throw e;
    }
    return topologyComponentBundle;
}

Example 12 with TopologyComponentBundle

use of com.hortonworks.streamline.streams.catalog.topology.TopologyComponentBundle in project streamline by hortonworks.

the class CustomProcessorInfo method toTopologyComponentBundle.

public TopologyComponentBundle toTopologyComponentBundle() throws IOException {
    TopologyComponentBundle result = new TopologyComponentBundle();
    result.setTimestamp(System.currentTimeMillis());
    result.setType(TopologyComponentBundle.TopologyComponentType.PROCESSOR);
    result.setSubType(TopologyLayoutConstants.JSON_KEY_CUSTOM_PROCESSOR_SUB_TYPE);
    result.setStreamingEngine(this.streamingEngine);
    if (TopologyLayoutConstants.STORM_STREAMING_ENGINE.equals(this.streamingEngine)) {
        result.setName("customProcessorBoltComponent");
    }
    result.setBuiltin(true);
    result.setTransformationClass("com.hortonworks.streamline.streams.layout.storm.CustomProcessorBoltFluxComponent");
    List<ComponentUISpecification.UIField> uiFields = new ArrayList<>();
    uiFields.addAll(getCustomProcessorUIFieldsWithPrefix());
    uiFields.add(this.createUIField(TopologyLayoutConstants.JSON_KEY_PARALLELISM, TopologyLayoutConstants.JSON_KEY_PARALLELISM, true, true, TopologyLayoutConstants.JSON_KEY_PARALLELISM_TOOLTIP, ComponentUISpecification.UIFieldType.NUMBER.NUMBER, 1));
    uiFields.add(this.createUIField(NAME, NAME, false, false, "Custom processor name", ComponentUISpecification.UIFieldType.STRING, this.name));
    uiFields.add(this.createUIField(DESCRIPTION, DESCRIPTION, false, false, "Custom processor description", ComponentUISpecification.UIFieldType.STRING, this.description));
    uiFields.add(this.createUIField(JAR_FILE_NAME, JAR_FILE_NAME, false, false, "Custom processor jar file", ComponentUISpecification.UIFieldType.STRING, this.jarFileName));
    uiFields.add(this.createUIField(CUSTOM_PROCESSOR_IMPL, CUSTOM_PROCESSOR_IMPL, false, false, "Custom processor interface implementation class", ComponentUISpecification.UIFieldType.STRING, this.customProcessorImpl));
    ObjectMapper objectMapper = new ObjectMapper();
    uiFields.add(this.createUIField(INPUT_SCHEMA, INPUT_SCHEMA, true, false, "Custom processor input schema", ComponentUISpecification.UIFieldType.STRING, objectMapper.writeValueAsString(this.inputSchema)));
    uiFields.add(this.createUIField(OUTPUT_SCHEMA, OUTPUT_SCHEMA, true, false, "Custom processor output schema", ComponentUISpecification.UIFieldType.STRING, objectMapper.writeValueAsString(this.outputSchema)));
    uiFields.add(this.createUIField(DIGEST, DIGEST, false, false, "MD5 digest of the jar file for this CP implementation", ComponentUISpecification.UIFieldType.STRING, this.digest));
    ComponentUISpecification componentUISpecification = new ComponentUISpecification();
    componentUISpecification.setFields(uiFields);
    result.setTopologyComponentUISpecification(componentUISpecification);
    return result;
}

Example 13 with TopologyComponentBundle

use of com.hortonworks.streamline.streams.catalog.topology.TopologyComponentBundle in project streamline by hortonworks.

the class StreamCatalogService method doImportTopology.

private Topology doImportTopology(Topology newTopology, TopologyData topologyData) throws Exception {
    List<TopologySource> topologySources = topologyData.getSources();
    Map<Long, Long> oldToNewComponentIds = new HashMap<>();
    Map<Long, Long> oldToNewRuleIds = new HashMap<>();
    Map<Long, Long> oldToNewWindowIds = new HashMap<>();
    Map<Long, Long> oldToNewBranchRuleIds = new HashMap<>();
    Map<Long, Long> oldToNewStreamIds = new HashMap<>();
    // import source streams
    for (TopologySource topologySource : topologySources) {
        topologySource.setOutputStreamIds(importOutputStreams(newTopology.getId(), oldToNewStreamIds, topologySource.getOutputStreams()));
        topologySource.setOutputStreams(null);
    }
    // import processor streams
    for (TopologyProcessor topologyProcessor : topologyData.getProcessors()) {
        topologyProcessor.setOutputStreamIds(importOutputStreams(newTopology.getId(), oldToNewStreamIds, topologyProcessor.getOutputStreams()));
        topologyProcessor.setOutputStreams(null);
    }
    // import rules
    for (TopologyRule rule : topologyData.getRules()) {
        Long currentId = rule.getId();
        rule.setId(null);
        TopologyRule addedRule = addRule(newTopology.getId(), rule);
        oldToNewRuleIds.put(currentId, addedRule.getId());
    }
    // import windowed rules
    for (TopologyWindow window : topologyData.getWindows()) {
        Long currentId = window.getId();
        window.setId(null);
        TopologyWindow addedWindow = addWindow(newTopology.getId(), window);
        oldToNewWindowIds.put(currentId, addedWindow.getId());
    }
    // import branch rules
    for (TopologyBranchRule branchRule : topologyData.getBranchRules()) {
        Long currentId = branchRule.getId();
        branchRule.setId(null);
        TopologyBranchRule addedBranchRule = addBranchRule(newTopology.getId(), branchRule);
        oldToNewBranchRuleIds.put(currentId, addedBranchRule.getId());
    }
    // import sources
    for (TopologySource topologySource : topologySources) {
        Long oldComponentId = topologySource.getId();
        topologySource.setId(null);
        topologySource.setTopologyId(newTopology.getId());
        TopologyComponentBundle bundle = getCurrentTopologyComponentBundle(TopologyComponentBundle.TopologyComponentType.SOURCE, topologyData.getBundleIdToType().get(topologySource.getTopologyComponentBundleId().toString()));
        topologySource.setTopologyComponentBundleId(bundle.getId());
        addTopologySource(newTopology.getId(), topologySource);
        oldToNewComponentIds.put(oldComponentId, topologySource.getId());
    }
    // import processors
    for (TopologyProcessor topologyProcessor : topologyData.getProcessors()) {
        Long oldComponentId = topologyProcessor.getId();
        topologyProcessor.setId(null);
        topologyProcessor.setTopologyId(newTopology.getId());
        TopologyComponentBundle bundle;
        String subType = topologyData.getBundleIdToType().get(topologyProcessor.getTopologyComponentBundleId().toString());
        if (TopologyLayoutConstants.JSON_KEY_CUSTOM_PROCESSOR_SUB_TYPE.equals(subType)) {
            QueryParam queryParam = new QueryParam(CustomProcessorInfo.NAME, topologyProcessor.getConfig().get(CustomProcessorInfo.NAME));
            Collection<TopologyComponentBundle> result = listCustomProcessorBundlesWithFilter(Collections.singletonList(queryParam));
            if (result.size() != 1) {
                throw new IllegalStateException("Not able to find topology component bundle for custom processor :" + topologyProcessor.getConfig().get(CustomProcessorInfo.NAME));
            }
            bundle = result.iterator().next();
        } else {
            bundle = getCurrentTopologyComponentBundle(TopologyComponentBundle.TopologyComponentType.PROCESSOR, subType);
        }
        topologyProcessor.setTopologyComponentBundleId(bundle.getId());
        Optional<Object> ruleListObj = topologyProcessor.getConfig().getAnyOptional(RulesProcessor.CONFIG_KEY_RULES);
        ruleListObj.ifPresent(ruleList -> {
            List<Long> ruleIds = new ObjectMapper().convertValue(ruleList, new TypeReference<List<Long>>() {
            });
            List<Long> updatedRuleIds = new ArrayList<>();
            if (ComponentTypes.RULE.equals(bundle.getSubType()) || ComponentTypes.PROJECTION.equals(bundle.getSubType())) {
                ruleIds.forEach(ruleId -> updatedRuleIds.add(oldToNewRuleIds.get(ruleId)));
            } else if (bundle.getSubType().equals(ComponentTypes.BRANCH)) {
                ruleIds.forEach(ruleId -> updatedRuleIds.add(oldToNewBranchRuleIds.get(ruleId)));
            } else if (bundle.getSubType().equals(ComponentTypes.WINDOW)) {
                ruleIds.forEach(ruleId -> updatedRuleIds.add(oldToNewWindowIds.get(ruleId)));
            }
            topologyProcessor.getConfig().setAny(RulesProcessor.CONFIG_KEY_RULES, updatedRuleIds);
        });
        addTopologyProcessor(newTopology.getId(), topologyProcessor);
        oldToNewComponentIds.put(oldComponentId, topologyProcessor.getId());
    }
    // import sinks
    for (TopologySink topologySink : topologyData.getSinks()) {
        topologySink.setTopologyId(newTopology.getId());
        Long currentId = topologySink.getId();
        topologySink.setId(null);
        TopologyComponentBundle bundle = getCurrentTopologyComponentBundle(TopologyComponentBundle.TopologyComponentType.SINK, topologyData.getBundleIdToType().get(topologySink.getTopologyComponentBundleId().toString()));
        topologySink.setTopologyComponentBundleId(bundle.getId());
        if (bundle.getSubType().equals(NOTIFICATION)) {
            updateNotifierJarFileName(topologySink);
        }
        addTopologySink(newTopology.getId(), topologySink);
        oldToNewComponentIds.put(currentId, topologySink.getId());
    }
    // import edges
    for (TopologyEdge topologyEdge : topologyData.getEdges()) {
        List<StreamGrouping> streamGroupings = topologyEdge.getStreamGroupings();
        for (StreamGrouping streamGrouping : streamGroupings) {
            Long newStreamId = oldToNewStreamIds.get(streamGrouping.getStreamId());
            streamGrouping.setStreamId(newStreamId);
        }
        topologyEdge.setId(null);
        topologyEdge.setTopologyId(newTopology.getId());
        topologyEdge.setFromId(oldToNewComponentIds.get(topologyEdge.getFromId()));
        topologyEdge.setToId(oldToNewComponentIds.get(topologyEdge.getToId()));
        addTopologyEdge(newTopology.getId(), topologyEdge);
    }
    // import topology editor metadata
    TopologyEditorMetadata topologyEditorMetadata = topologyData.getTopologyEditorMetadata();
    topologyEditorMetadata.setTopologyId(newTopology.getId());
    if (topologyEditorMetadata.getData() != null) {
        TopologyUIData topologyUIData = new ObjectMapper().readValue(topologyEditorMetadata.getData(), TopologyUIData.class);
        topologyUIData.getSources().forEach(c -> c.setId(oldToNewComponentIds.get(c.getId())));
        topologyUIData.getProcessors().forEach(c -> c.setId(oldToNewComponentIds.get(c.getId())));
        topologyUIData.getSinks().forEach(c -> c.setId(oldToNewComponentIds.get(c.getId())));
        topologyEditorMetadata.setData(new ObjectMapper().writeValueAsString(topologyUIData));
    } else {
        topologyEditorMetadata.setData(StringUtils.EMPTY);
    }
    addTopologyEditorMetadata(newTopology.getId(), topologyData.getTopologyEditorMetadata());
    return newTopology;
}

Example 14 with TopologyComponentBundle

use of com.hortonworks.streamline.streams.catalog.topology.TopologyComponentBundle in project streamline by hortonworks.

the class StreamCatalogService method setReconfigureRules.

private void setReconfigureRules(List<TopologyProcessor> processors, List<TopologyStream> affectedStreams) {
    Map<Long, BiFunction<TopologyProcessor, Long, BaseTopologyRule>> bundles = new HashMap<>();
    TopologyComponentBundle bundle = getCurrentTopologyComponentBundle(TopologyComponentBundle.TopologyComponentType.PROCESSOR, ComponentTypes.RULE);
    bundles.put(bundle.getId(), (p, r) -> getRule(p.getTopologyId(), r, p.getVersionId()));
    bundle = getCurrentTopologyComponentBundle(TopologyComponentBundle.TopologyComponentType.PROCESSOR, ComponentTypes.BRANCH);
    bundles.put(bundle.getId(), (p, r) -> getBranchRule(p.getTopologyId(), r, p.getVersionId()));
    bundle = getCurrentTopologyComponentBundle(TopologyComponentBundle.TopologyComponentType.PROCESSOR, ComponentTypes.PROJECTION);
    bundles.put(bundle.getId(), (p, r) -> getRule(p.getTopologyId(), r, p.getVersionId()));
    bundle = getCurrentTopologyComponentBundle(TopologyComponentBundle.TopologyComponentType.PROCESSOR, ComponentTypes.WINDOW);
    bundles.put(bundle.getId(), (p, r) -> getWindow(p.getTopologyId(), r, p.getVersionId()));
    Set<String> affectedStreamIds = affectedStreams.stream().map(TopologyStream::getStreamId).collect(Collectors.toSet());
    for (TopologyProcessor processor : processors) {
        BiFunction<TopologyProcessor, Long, BaseTopologyRule> function;
        if ((function = bundles.get(processor.getTopologyComponentBundleId())) != null) {
            Optional<Object> ruleList = processor.getConfig().getAnyOptional(RulesProcessor.CONFIG_KEY_RULES);
            if (ruleList.isPresent()) {
                ObjectMapper objectMapper = new ObjectMapper();
                List<Long> ruleIds = objectMapper.convertValue(ruleList.get(), new TypeReference<List<Long>>() {
                });
                for (Long ruleId : ruleIds) {
                    BaseTopologyRule rule = function.apply(processor, ruleId);
                    if (rule != null) {
                        for (String stream : rule.getInputStreams()) {
                            if (affectedStreamIds.contains(stream)) {
                                rule.setReconfigure(true);
                                dao.addOrUpdate(rule);
                                break;
                            }
                        }
                    }
                }
            }
        }
    }
}

Example 15 with TopologyComponentBundle

use of com.hortonworks.streamline.streams.catalog.topology.TopologyComponentBundle in project streamline by hortonworks.

the class StreamCatalogService method removeCustomProcessorInfoAsBundle.

public CustomProcessorInfo removeCustomProcessorInfoAsBundle(String name) throws IOException {
    List<QueryParam> queryParams = new ArrayList<>();
    queryParams.add(new QueryParam(CustomProcessorInfo.NAME, name));
    Collection<TopologyComponentBundle> result = this.listCustomProcessorBundlesWithFilter(queryParams);
    if (result.isEmpty() || result.size() != 1) {
        throw new IOException("Failed to delete custom processor with name:" + name);
    }
    TopologyComponentBundle customProcessorBundle = result.iterator().next();
    Collection<TopologyProcessor> processors = this.listTopologyProcessors();
    if (processors != null && !processors.isEmpty()) {
        for (TopologyProcessor topologyProcessor : processors) {
            if (topologyProcessor.getTopologyComponentBundleId().equals(customProcessorBundle.getId())) {
                throw new IOException("Cannot delete custom processor as it is being used in one of the topologies.");
            }
        }
    }
    this.removeTopologyComponentBundle(customProcessorBundle.getId());
    return CustomProcessorInfo.fromTopologyComponentBundle(customProcessorBundle);
}