Examples with UploadProgressMessage com.thinkbiganalytics.feedmgr.rest.model.UploadProgressMessage used on opensource projects

Example 11 with UploadProgressMessage

use of com.thinkbiganalytics.feedmgr.rest.model.UploadProgressMessage in project kylo by Teradata.

the class ImportFeedTemplateXml method importTemplate.

public boolean importTemplate() {
    if (importTemplate.isValid()) {
        UploadProgressMessage importStatusMessage = start();
        boolean createReusableFlow = importTemplateOptions.isImport(ImportComponent.REUSABLE_TEMPLATE);
        boolean overwrite = importTemplateOptions.isImportAndOverwrite(ImportComponent.NIFI_TEMPLATE);
        log.info("Importing XML file template {}, overwrite: {}, reusableFlow: {}", importTemplate.getFileName(), overwrite, createReusableFlow);
        boolean valid = importIntoNiFiAndCreateInstance();
        if (valid) {
            valid &= connectAndValidate();
            if (valid) {
                valid &= validateInstance();
            }
            if (valid) {
                importStatusMessage.update("Validated " + importTemplate.getTemplateName(), true);
                cleanup();
            } else {
                rollback();
                importStatusMessage.update("An error occurred importing the flow, " + importTemplate.getTemplateName(), false);
            }
        }
    }
    log.info("Import NiFi Template for {} finished", importTemplate.getFileName());
    return importTemplate.isValid() && importTemplate.isSuccess();
}

Example 12 with UploadProgressMessage

use of com.thinkbiganalytics.feedmgr.rest.model.UploadProgressMessage in project kylo by Teradata.

the class ImportReusableTemplate method connect.

private boolean connect() {
    NifiProcessGroup processGroup = this.newTemplateInstance;
    UploadProgressMessage importStatusMessage = uploadProgressService.addUploadStatus(importTemplateOptions.getUploadKey(), "Connecting and validating components for " + importTemplate.getTemplateName());
    connectReusableFlow(importStatusMessage, processGroup);
    recreateOutputPortConnections(importStatusMessage, processGroup);
    boolean valid = validateOutputPortConnections(processGroup);
    importStatusMessage.update("Connected and validated components for " + importTemplate.getTemplateName(), valid);
    // create any remote process group ports and connect them on the main NiFi canvas
    if (isClustered()) {
        ImportComponentOption remoteProcessGroupOption = importTemplateOptions.findImportComponentOption(ImportComponent.REMOTE_INPUT_PORT);
        if (remoteProcessGroupOption.isUserAcknowledged() && remoteProcessGroupOption.isShouldImport()) {
            if (remoteProcessGroupOption.getRemoteProcessGroupInputPortsForTemplate(importTemplate.getTemplateName()).stream().anyMatch(inputPort -> inputPort.isSelected())) {
                UploadProgressMessage remoteInputPortsMessage = uploadProgressService.addUploadStatus(importTemplateOptions.getUploadKey(), "Creating remote input port connections for " + remoteProcessGroupOption.getRemoteProcessGroupInputPortsForTemplate(importTemplate.getTemplateName()).stream().filter(inputPort -> inputPort.isSelected()).map(inputPort -> inputPort.getInputPortName()).collect(Collectors.joining(",")));
                valid &= createRemoteInputPorts(remoteInputPortsMessage);
            }
        }
        if (remoteProcessGroupOption.isShouldImport() && remoteProcessGroupOption.isUserAcknowledged()) {
            // identify if the user wished to remove any input ports.
            valid = removeConnectionsAndInputs();
        }
    }
    return valid && newTemplateInstance.isSuccess();
}

Example 13 with UploadProgressMessage

use of com.thinkbiganalytics.feedmgr.rest.model.UploadProgressMessage in project kylo by Teradata.

the class ImportReusableTemplate method createRemoteInputPorts.

private boolean createRemoteInputPorts(UploadProgressMessage remoteInputPortsMessage) {
    ImportComponentOption remoteProcessGroupOption = importTemplateOptions.findImportComponentOption(ImportComponent.REMOTE_INPUT_PORT);
    String rootProcessGroupId = templateConnectionUtil.getRootProcessGroup().getId();
    String reusableTemplateProcessGroupId = templateConnectionUtil.getReusableTemplateProcessGroupId();
    Map<String, PortDTO> reusableTemplateCategoryPorts = getReusableTemplateCategoryProcessGroup().getContents().getInputPorts().stream().collect(Collectors.toMap(p -> p.getName(), p -> p));
    StringBuffer connectedStr = new StringBuffer("");
    remoteProcessGroupOption.getRemoteProcessGroupInputPortsForTemplate(importTemplate.getTemplateName()).stream().filter(r -> r.isSelected() && !r.isExisting()).forEach(r -> {
        // check/create the port at the parent canvas
        PortDTO portDTO = new PortDTO();
        portDTO.setName(r.getInputPortName());
        portDTO.setType(NifiConstants.INPUT_PORT);
        portDTO.setState(NifiProcessUtil.PROCESS_STATE.STOPPED.name());
        PortDTO newInputPort = nifiRestClient.getNiFiRestClient().processGroups().createInputPort(rootProcessGroupId, portDTO);
        getItemsCreated().addCreatedRemoteInputPort(newInputPort);
        PortDTO reusableTemplatePort = reusableTemplateCategoryPorts.get(r.getInputPortName());
        // connect this to the Reusable Template input port with the same name
        ConnectableDTO source = new ConnectableDTO();
        source.setGroupId(rootProcessGroupId);
        source.setId(newInputPort.getId());
        source.setName(newInputPort.getName());
        source.setType(NifiConstants.NIFI_PORT_TYPE.INPUT_PORT.name());
        ConnectableDTO dest = new ConnectableDTO();
        dest.setGroupId(reusableTemplateProcessGroupId);
        dest.setName(r.getInputPortName());
        dest.setId(reusableTemplatePort.getId());
        dest.setType(NifiConstants.NIFI_PORT_TYPE.INPUT_PORT.name());
        ConnectionDTO connectionDTO = nifiRestClient.getNiFiRestClient().processGroups().createConnection(rootProcessGroupId, source, dest);
        getItemsCreated().addCreatedRemoteInputPortConnection(connectionDTO);
        if (connectedStr.length() != 0) {
            connectedStr.append(",");
        } else {
            connectedStr.append("Created ");
        }
        connectedStr.append(r.getInputPortName());
        remoteInputPortsMessage.update(connectedStr.toString());
    });
    if (connectedStr.length() != 0) {
        connectedStr.append(" as remote input ports");
    }
    if (connectedStr.length() > 0) {
        remoteInputPortsMessage.update(connectedStr.toString(), true);
    } else {
        remoteInputPortsMessage.complete(true);
    }
    return true;
}

Example 14 with UploadProgressMessage

use of com.thinkbiganalytics.feedmgr.rest.model.UploadProgressMessage in project kylo by Teradata.

the class ImportReusableTemplate method removeConnectionsAndInputs.

private boolean removeConnectionsAndInputs() {
    Optional<TemplateRemoteInputPortConnections> existingRemoteProcessInputPortInformation = getExistingRemoteProcessInputPortInformation();
    if (existingRemoteProcessInputPortInformation.isPresent()) {
        // find the input ports that are 'existing' but not 'selected' .  These should be deleted
        Set<String> inputPortNamesToRemove = remoteProcessGroupInputPortMap.values().stream().filter((remoteInputPort -> !remoteInputPort.isSelected() && existingRemoteProcessInputPortInformation.get().getExistingRemoteInputPortNames().contains(remoteInputPort.getInputPortName()))).map(remoteInputPort -> remoteInputPort.getInputPortName()).collect(Collectors.toSet());
        // Find the connections that match the input ports that are to be removed
        Set<ConnectionDTO> connectionsToRemove = existingRemoteProcessInputPortInformation.get().getExistingRemoteConnectionsToTemplate().stream().filter(connectionDTO -> inputPortNamesToRemove.contains(connectionDTO.getSource().getName())).collect(Collectors.toSet());
        log.info("Removing input ports {}", inputPortNamesToRemove);
        Set<ConnectionDTO> connectionsWithQueue = new HashSet<>();
        // first validate the queues are empty ... if not warn user the following ports cant be deleted and rollback
        connectionsToRemove.stream().forEach(connection -> {
            Optional<ConnectionStatusEntity> connectionStatus = nifiRestClient.getNiFiRestClient().connections().getConnectionStatus(connection.getId());
            if (connectionStatus.isPresent() && connectionStatus.get().getConnectionStatus().getAggregateSnapshot().getFlowFilesQueued() > 0) {
                connectionsWithQueue.add(connection);
            }
        });
        if (!connectionsWithQueue.isEmpty()) {
            UploadProgressMessage importStatusMessage = uploadProgressService.addUploadStatus(importTemplateOptions.getUploadKey(), "Unable to remove inputPort and connection for :" + connectionsWithQueue.stream().map(c -> c.getSource().getName()).collect(Collectors.joining(",")) + ". The Queues are not empty. Failed to import template: " + importTemplate.getTemplateName(), true, false);
            importTemplate.setValid(false);
            importTemplate.setSuccess(false);
            return false;
        } else {
            connectionsToRemove.stream().forEach(connection -> {
                nifiRestClient.deleteConnection(connection, false);
                getItemsCreated().addDeletedRemoteInputPortConnection(connection);
                try {
                    PortDTO deletedPort = nifiRestClient.getNiFiRestClient().ports().deleteInputPort(connection.getSource().getId());
                    if (deletedPort != null) {
                        getItemsCreated().addDeletedRemoteInputPort(deletedPort);
                    }
                } catch (NifiComponentNotFoundException e) {
                // this is ok to catch as its deleted already
                }
            });
            UploadProgressMessage importStatusMessage = uploadProgressService.addUploadStatus(importTemplateOptions.getUploadKey(), "Removed inputPort and connection for :" + connectionsToRemove.stream().map(c -> c.getSource().getName()).collect(Collectors.joining(",")) + " for template: " + importTemplate.getTemplateName(), true, true);
        }
        return true;
    }
    return true;
}

Example 15 with UploadProgressMessage

use of com.thinkbiganalytics.feedmgr.rest.model.UploadProgressMessage in project kylo by Teradata.

the class ImportTemplateArchive method rollback.

public boolean rollback() {
    UploadProgressMessage statusMessage = uploadProgressService.addUploadStatus(this.importTemplateOptions.getUploadKey(), "Errors were found registering the template.  Attempting to rollback.");
    if (this.existingTemplate != null) {
        rollbackRegisteredTemplate(existingTemplate);
    }
    rollbackReusableTemplates();
    UploadProgressMessage rollbackStatus = super.restoreOldTemplateXml();
    rollbackStatus.complete(true);
    return true;
}