Examples with NifiClientRuntimeException com.thinkbiganalytics.nifi.rest.client.NifiClientRuntimeException used on opensource projects

Example 1 with NifiClientRuntimeException

use of com.thinkbiganalytics.nifi.rest.client.NifiClientRuntimeException in project kylo by Teradata.

the class TemplateCreationHelper method versionProcessGroup.

/**
 * Version a ProcessGroup renaming it with the name - {timestamp millis}.
 * If {@code removeIfInactive} is true it will not version but just delete it
 *
 * @param processGroup the group to version
 */
public VersionedProcessGroup versionProcessGroup(ProcessGroupDTO processGroup, String versionIdentifier) {
    log.info("Versioning Process Group {} ", processGroup.getName());
    VersionedProcessGroup versionedProcessGroup = new VersionedProcessGroup();
    versionedProcessGroup.setProcessGroupPriorToVersioning(processGroup);
    versionedProcessGroup.setProcessGroupName(processGroup.getName());
    List<ProcessorDTO> inputProcessorsPriorToDisabling = restClient.disableAllInputProcessors(processGroup.getId());
    versionedProcessGroup.setInputProcessorsPriorToDisabling(inputProcessorsPriorToDisabling);
    log.info("Disabled Inputs for {} ", processGroup.getName());
    // attempt to stop all processors
    try {
        restClient.stopInputs(processGroup.getId());
        log.info("Stopped Input Ports for {}, ", processGroup.getName());
    } catch (Exception e) {
        log.error("Error trying to stop Input Ports for {} while creating a new version ", processGroup.getName());
    }
    // delete input connections
    try {
        List<ConnectionDTO> deletedConnections = deleteInputPortConnections(processGroup);
        versionedProcessGroup.setDeletedInputPortConnections(deletedConnections);
    } catch (NifiClientRuntimeException e) {
        log.error("Error trying to delete input port connections for Process Group {} while creating a new version. ", processGroup.getName(), e);
        getErrors().add(new NifiError(NifiError.SEVERITY.FATAL, "The input port connections to the process group " + processGroup.getName() + " could not be deleted. Please delete them manually " + "in NiFi and try again."));
    }
    String versionedProcessGroupName = getVersionedProcessGroupName(processGroup.getName(), versionIdentifier);
    versionedProcessGroup.setVersionedProcessGroupName(versionedProcessGroupName);
    // rename the feedGroup to be name+timestamp
    processGroup.setName(versionedProcessGroupName);
    restClient.updateProcessGroup(processGroup);
    log.info("Renamed ProcessGroup to  {}, ", processGroup.getName());
    versionedProcessGroup.setVersionedProcessGroup(processGroup);
    return versionedProcessGroup;
}

Example 2 with NifiClientRuntimeException

use of com.thinkbiganalytics.nifi.rest.client.NifiClientRuntimeException in project kylo by Teradata.

the class TemplateCreationHelper method updateControllerServiceReferences.

/**
 * Fix references to the controller services on the processor properties
 *
 * @param processors                  processors to inspect
 * @param controllerServiceProperties property overrides for controller services
 * @return the list of properties that were modified
 */
public List<NifiProperty> updateControllerServiceReferences(List<ProcessorDTO> processors, Map<String, String> controllerServiceProperties, TemplateInstance instance) {
    try {
        processors = reassignControllerServiceIds(processors, instance);
        // merge the snapshotted services with the newly created ones and update respective processors in the newly created flow
        final Map<String, ControllerServiceDTO> enabledServices = new HashMap<>();
        Map<String, ControllerServiceDTO> allServices = mergedControllerServices;
        for (ControllerServiceDTO dto : allServices.values()) {
            if (NifiProcessUtil.SERVICE_STATE.ENABLED.name().equals(dto.getState())) {
                enabledServices.put(dto.getId(), dto);
                enabledServices.put(dto.getName(), dto);
            }
        }
        List<NifiProperty> properties = new ArrayList<>();
        Map<String, ProcessGroupDTO> processGroupDTOMap = new HashMap<>();
        for (ProcessorDTO dto : processors) {
            ProcessGroupDTO groupDTO = processGroupDTOMap.get(dto.getParentGroupId());
            if (groupDTO == null) {
                // we can create a tmp group dto here as all we need is the id
                groupDTO = new ProcessGroupDTO();
                groupDTO.setId(dto.getParentGroupId());
                groupDTO.setName(dto.getParentGroupId());
                processGroupDTOMap.put(dto.getParentGroupId(), groupDTO);
            }
            properties.addAll(NifiPropertyUtil.getPropertiesForProcessor(groupDTO, dto, restClient.getPropertyDescriptorTransform()));
        }
        List<NifiProperty> updatedProperties = fixControllerServiceReferences(controllerServiceProperties, enabledServices, allServices, properties);
        updatedProperties.forEach(property -> restClient.updateProcessorProperty(property.getProcessGroupId(), property.getProcessorId(), property));
        return updatedProperties;
    } catch (NifiClientRuntimeException e) {
        errors.add(new NifiError(NifiError.SEVERITY.FATAL, "Error trying to identify Controller Services. " + e.getMessage(), NifiProcessGroup.CONTROLLER_SERVICE_CATEGORY));
    }
    return Collections.emptyList();
}

Example 3 with NifiClientRuntimeException

use of com.thinkbiganalytics.nifi.rest.client.NifiClientRuntimeException in project kylo by Teradata.

the class TemplateCreationHelper method deleteInputPortConnections.

/**
 * Deletes the input port connections to the specified process group.
 *
 * <p>When versioning we want to delete only the input port connections. Keep output port connections in place as they may still have data running through them that should flow through the
 * system.</p>
 *
 * @param processGroup the process group with input port connections
 * @throws NifiClientRuntimeException if a connection cannot be deleted
 */
private List<ConnectionDTO> deleteInputPortConnections(@Nonnull final ProcessGroupDTO processGroup) throws NifiClientRuntimeException {
    // Get the list of incoming connections coming from some source to this process group
    List<ConnectionDTO> deletedConnections = new ArrayList<>();
    final Set<ConnectionDTO> connectionsEntity = nifiObjectCache.isCacheConnections() ? nifiObjectCache.getConnections(processGroup.getParentGroupId()) : restClient.getProcessGroupConnections(processGroup.getParentGroupId());
    if (connectionsEntity == null) {
        return deletedConnections;
    }
    final List<ConnectionDTO> connections = NifiConnectionUtil.findConnectionsMatchingDestinationGroupId(connectionsEntity, processGroup.getId());
    if (connections == null) {
        return deletedConnections;
    }
    Set<String> removedConnections = new HashSet<>();
    // Delete the connections
    for (ConnectionDTO connection : connections) {
        final String type = connection.getSource().getType();
        log.info("Found connection {} matching source type {} and destination group {}.", connection.getId(), type, connection.getDestination().getId());
        // Stop the port
        if (NifiConstants.NIFI_PORT_TYPE.INPUT_PORT.name().equalsIgnoreCase(type)) {
            try {
                restClient.stopInputPort(connection.getSource().getGroupId(), connection.getSource().getId());
                log.info("Stopped input port {} for connection: {} ", connection.getSource().getId(), connection.getId());
            } catch (Exception e) {
                log.error("Failed to stop input port for connection: {}", connection.getId(), e);
                throw new NifiClientRuntimeException("Error stopping the input port " + connection.getSource().getId() + " for connection " + connection.getId() + " prior to deleting the " + "connection.");
            }
        }
        // Delete the connection
        try {
            restClient.deleteConnection(connection, false);
            removedConnections.add(connection.getId());
            deletedConnections.add(connection);
        } catch (Exception e) {
            log.error("Failed to delete the connection: {}", connection.getId(), e);
            final String source = (connection.getSource() != null) ? connection.getSource().getName() : null;
            final String destination = (connection.getDestination() != null) ? connection.getDestination().getName() : null;
            throw new NifiClientRuntimeException("Error deleting the connection " + connection.getId() + " with source " + source + " and destination " + destination + ".");
        }
    }
    nifiObjectCache.removeConnections(processGroup.getParentGroupId(), removedConnections);
    return deletedConnections;
}

Example 4 with NifiClientRuntimeException

use of com.thinkbiganalytics.nifi.rest.client.NifiClientRuntimeException in project kylo by Teradata.

the class TemplateCreationHelper method mergeControllerServices.

private void mergeControllerServices(TemplateInstance templateInstance) {
    final Map<String, ControllerServiceDTO> map = new HashMap<String, ControllerServiceDTO>();
    final Map<String, List<ControllerServiceDTO>> serviceNameMap = new HashMap<>();
    // first use the snapshotted servies as a baseline
    for (ControllerServiceDTO serviceDTO : snapshotControllerServices) {
        map.put(serviceDTO.getId(), serviceDTO);
        if (!serviceNameMap.containsKey(serviceDTO.getName())) {
            serviceNameMap.put(serviceDTO.getName(), new ArrayList<ControllerServiceDTO>());
        }
        serviceNameMap.get(serviceDTO.getName()).add(serviceDTO);
    }
    java.util.function.Predicate<ControllerServiceDTO> matchingServiceFilter = (cs) -> map.containsKey(cs.getId()) || serviceNameMap.containsKey(cs.getName());
    List<ControllerServiceDTO> matchingControllerServices = newlyCreatedControllerServices.stream().filter(matchingServiceFilter).collect(Collectors.toList());
    List<ControllerServiceDTO> unmatchedServices = newlyCreatedControllerServices.stream().filter(matchingServiceFilter.negate()).collect(Collectors.toList());
    // if the service has additional propertyDescriptors that identify other services we need to fetch the service by its id.
    if (unmatchedServices != null && !unmatchedServices.isEmpty()) {
        Map<String, ControllerServiceDTO> updatedServices = unmatchedServices.stream().map(serviceToAdd -> {
            // if the service has additional propertyDescriptors that identify other services we need to fetch the service by its id
            if (serviceToAdd.getDescriptors() != null && serviceToAdd.getDescriptors().values().stream().anyMatch(propertyDescriptorDTO -> StringUtils.isNotBlank(propertyDescriptorDTO.getIdentifiesControllerService()))) {
                try {
                    Optional<ControllerServiceDTO> cs = restClient.getNiFiRestClient().controllerServices().findById(serviceToAdd.getId());
                    if (cs.isPresent()) {
                        return cs.get();
                    } else {
                        return serviceToAdd;
                    }
                } catch (Exception e) {
                    return serviceToAdd;
                }
            } else {
                return serviceToAdd;
            }
        }).collect(Collectors.toMap(service -> service.getId(), service -> service));
        map.putAll(updatedServices);
        // update the core item
        newlyCreatedControllerServices = newlyCreatedControllerServices.stream().map(controllerServiceDTO -> {
            if (map.containsKey(controllerServiceDTO.getId())) {
                return updatedServices.get(controllerServiceDTO.getId());
            } else {
                return controllerServiceDTO;
            }
        }).collect(Collectors.toSet());
    }
    // if match existing services, then delete the new ones
    if (matchingControllerServices != null && !matchingControllerServices.isEmpty()) {
        for (ControllerServiceDTO serviceToDelete : matchingControllerServices) {
            try {
                if (templateInstance != null) {
                    templateInstance.addDeletedServiceMapping(serviceToDelete.getId(), serviceNameMap.get(serviceToDelete.getName()));
                }
                restClient.deleteControllerService(serviceToDelete.getId());
            } catch (NifiClientRuntimeException e) {
                log.error("Exception while attempting to mergeControllerServices.  Unable to delete Service {}. {}", serviceToDelete.getId(), e.getMessage());
            }
        }
    }
    mergedControllerServices = map;
    // validate
    // Create a map of the Controller Service Name to list of matching services
    this.serviceNameMap = mergedControllerServices.values().stream().collect(Collectors.groupingBy(cs -> cs.getName()));
    this.enabledServiceNameMap = mergedControllerServices.values().stream().filter(cs -> NifiProcessUtil.SERVICE_STATE.ENABLED.name().equalsIgnoreCase(cs.getState())).collect(Collectors.groupingBy(cs -> cs.getName()));
}

Example 5 with NifiClientRuntimeException

use of com.thinkbiganalytics.nifi.rest.client.NifiClientRuntimeException in project kylo by Teradata.

the class DatasourceModelTransform method updateDomain.

/**
 * Updates the specified domain object with properties from the specified REST object.
 *
 * @param domain the domain object
 * @param ds     the REST object
 */
private void updateDomain(@Nonnull final com.thinkbiganalytics.metadata.api.datasource.UserDatasource domain, @Nonnull final JdbcDatasource ds) {
    updateDomain(domain, (UserDatasource) ds);
    domain.getDetails().map(JdbcDatasourceDetails.class::cast).ifPresent(details -> {
        // Look for changed properties
        final Map<String, String> properties = new HashMap<>();
        if (StringUtils.isNotBlank(ds.getDatabaseConnectionUrl())) {
            properties.put(DatasourceConstants.DATABASE_CONNECTION_URL, ds.getDatabaseConnectionUrl());
        }
        if (StringUtils.isNotBlank(ds.getDatabaseDriverClassName())) {
            properties.put(DatasourceConstants.DATABASE_DRIVER_CLASS_NAME, ds.getDatabaseDriverClassName());
        }
        if (ds.getDatabaseDriverLocation() != null) {
            properties.put(DatasourceConstants.DATABASE_DRIVER_LOCATION, ds.getDatabaseDriverLocation());
        }
        if (ds.getDatabaseUser() != null) {
            properties.put(DatasourceConstants.DATABASE_USER, ds.getDatabaseUser());
        }
        if (ds.getPassword() != null) {
            details.setPassword(encryptor.encrypt(ds.getPassword()));
            properties.put(DatasourceConstants.PASSWORD, StringUtils.isNotEmpty(ds.getPassword()) ? ds.getPassword() : null);
        }
        // Update or create the controller service
        ControllerServiceDTO controllerService = null;
        if (details.getControllerServiceId().isPresent()) {
            controllerService = new ControllerServiceDTO();
            controllerService.setId(details.getControllerServiceId().get());
            controllerService.setName(ds.getName());
            controllerService.setComments(ds.getDescription());
            controllerService.setProperties(properties);
            try {
                controllerService = nifiRestClient.controllerServices().updateServiceAndReferencingComponents(controllerService);
                ds.setControllerServiceId(controllerService.getId());
            } catch (final NifiComponentNotFoundException e) {
                log.warn("Controller service is missing for datasource: {}", domain.getId(), e);
                controllerService = null;
            }
        }
        if (controllerService == null) {
            controllerService = new ControllerServiceDTO();
            controllerService.setType("org.apache.nifi.dbcp.DBCPConnectionPool");
            controllerService.setName(ds.getName());
            controllerService.setComments(ds.getDescription());
            controllerService.setProperties(properties);
            final ControllerServiceDTO newControllerService = nifiRestClient.controllerServices().create(controllerService);
            try {
                nifiRestClient.controllerServices().updateStateById(newControllerService.getId(), NiFiControllerServicesRestClient.State.ENABLED);
            } catch (final NifiClientRuntimeException nifiException) {
                log.error("Failed to enable controller service for datasource: {}", domain.getId(), nifiException);
                nifiRestClient.controllerServices().disableAndDeleteAsync(newControllerService.getId());
                throw nifiException;
            }
            details.setControllerServiceId(newControllerService.getId());
            ds.setControllerServiceId(newControllerService.getId());
        }
    });
}