use of com.thinkbiganalytics.nifi.rest.model.NifiProperty in project kylo by Teradata.
the class TemplateInstanceCreator method createTemplate.
public NifiProcessGroup createTemplate() throws TemplateCreationException {
try {
NifiProcessGroup newProcessGroup = null;
TemplateDTO template = restClient.getTemplateById(templateId);
VersionedProcessGroup versionedProcessGroup = null;
if (template != null) {
TemplateCreationHelper templateCreationHelper = new TemplateCreationHelper(this.restClient);
templateCreationHelper.setTemplateProperties(templateProperties);
String processGroupId = null;
ProcessGroupDTO group = null;
if (isCreateReusableFlow()) {
log.info("Creating a new Reusable flow instance for template: {} ", template.getName());
// 1 get/create the parent "reusable_templates" processgroup
ProcessGroupDTO reusableParentGroup = restClient.getProcessGroupByName("root", TemplateCreationHelper.REUSABLE_TEMPLATES_PROCESS_GROUP_NAME);
if (reusableParentGroup == null) {
reusableParentGroup = restClient.createProcessGroup("root", TemplateCreationHelper.REUSABLE_TEMPLATES_PROCESS_GROUP_NAME);
}
ProcessGroupDTO thisGroup = restClient.getProcessGroupByName(reusableParentGroup.getId(), template.getName());
if (thisGroup != null) {
// version the group
log.info("A previous Process group of with this name {} was found. Versioning it before continuing", thisGroup.getName());
versionedProcessGroup = templateCreationHelper.versionProcessGroup(thisGroup, this.getVersionIdentifier());
}
group = restClient.createProcessGroup(reusableParentGroup.getId(), template.getName());
} else {
String tmpName = template.getName() + "_" + System.currentTimeMillis();
group = restClient.createProcessGroup(tmpName);
log.info("Creating a temporary process group with name {} for template {} ", tmpName, template.getName());
}
processGroupId = group.getId();
if (StringUtils.isNotBlank(processGroupId)) {
// snapshot the existing controller services
templateCreationHelper.snapshotControllerServiceReferences();
log.info("Successfully Snapshot of controller services");
// create the flow from the template
TemplateInstance instance = templateCreationHelper.instantiateFlowFromTemplate(processGroupId, templateId);
FlowSnippetDTO flowSnippetDTO = instance.getFlowSnippetDTO();
log.info("Successfully created the temp flow");
if (this.createReusableFlow) {
ensureInputPortsForReuseableTemplate(processGroupId);
log.info("Reusable flow, input ports created successfully.");
}
// mark the new services that were created as a result of creating the new flow from the template
templateCreationHelper.identifyNewlyCreatedControllerServiceReferences(instance);
ProcessGroupDTO entity = restClient.getProcessGroup(processGroupId, true, true);
// replace static properties and inject values into the flow
List<NifiProperty> processorProperties = NifiPropertyUtil.getProperties(entity, restClient.getPropertyDescriptorTransform());
if (processorProperties != null) {
boolean didReplace = false;
for (NifiProperty property : processorProperties) {
boolean replaced = ConfigurationPropertyReplacer.resolveStaticConfigurationProperty(property, staticConfigPropertyMap);
if (replaced) {
// update the properties that are replaced
if (property.getPropertyDescriptor() != null && StringUtils.isNotBlank(property.getPropertyDescriptor().getIdentifiesControllerService())) {
// verify the property is a valid cs property
String value = property.getValue();
if (templateCreationHelper.getEnabledServiceNameMap().containsKey(value)) {
property.setValue(templateCreationHelper.getEnabledServiceNameMap().get(value).get(0).getId());
}
}
restClient.updateProcessorProperty(property.getProcessGroupId(), property.getProcessorId(), property);
didReplace = true;
}
}
// if we replaced any properties, refetch to get the latest data
if (didReplace) {
entity = restClient.getProcessGroup(processGroupId, true, true);
}
}
// identify the various processors (first level initial processors)
List<ProcessorDTO> inputProcessors = NifiProcessUtil.getInputProcessors(entity);
ProcessorDTO input = null;
List<ProcessorDTO> nonInputProcessors = NifiProcessUtil.getNonInputProcessors(entity);
// if the input is null attempt to get the first input available on the template
if (input == null && inputProcessors != null && !inputProcessors.isEmpty()) {
input = inputProcessors.get(0);
}
log.info("Attempt to update/validate controller services for template.");
// update any references to the controller services and try to assign the value to an enabled service if it is not already
boolean updatedControllerServices = false;
if (input != null) {
log.info("attempt to update controllerservices on {} input processor ", input.getName());
List<NifiProperty> updatedProperties = templateCreationHelper.updateControllerServiceReferences(Lists.newArrayList(inputProcessors), staticConfigPropertyStringMap, instance);
updatedControllerServices = !updatedProperties.isEmpty();
}
log.info("attempt to update controllerservices on {} processors ", (nonInputProcessors != null ? nonInputProcessors.size() : 0));
List<NifiProperty> updatedProperties = templateCreationHelper.updateControllerServiceReferences(nonInputProcessors, staticConfigPropertyStringMap, instance);
if (!updatedControllerServices) {
updatedControllerServices = !updatedProperties.isEmpty();
}
log.info("Controller service validation complete");
// refetch processors for updated errors
entity = restClient.getProcessGroup(processGroupId, true, true);
nonInputProcessors = NifiProcessUtil.getNonInputProcessors(entity);
inputProcessors = NifiProcessUtil.getInputProcessors(entity);
if (inputProcessors != null && !inputProcessors.isEmpty()) {
input = inputProcessors.get(0);
}
// /make the input/output ports in the category group as running
if (isCreateReusableFlow()) {
log.info("Reusable flow, attempt to mark the connection ports as running.");
templateCreationHelper.startProcessGroupAndParentInputPorts(entity);
// templateCreationHelper.markConnectionPortsAsRunning(entity);
log.info("Reusable flow. Successfully marked the ports as running.");
}
newProcessGroup = new NifiProcessGroup(entity, input, nonInputProcessors);
newProcessGroup.setVersionedProcessGroup(versionedProcessGroup);
newProcessGroup.setReusableFlowInstance(isCreateReusableFlow());
if (isCreateReusableFlow()) {
// call listeners notify of before mark as running processing
if (creationCallback != null) {
try {
creationCallback.beforeMarkAsRunning(template.getName(), entity);
} catch (Exception e) {
log.error("Error calling callback beforeMarkAsRunning ", e);
}
}
log.info("Reusable flow, attempt to mark the Processors as running.");
templateCreationHelper.markProcessorsAsRunning(newProcessGroup);
log.info("Reusable flow. Successfully marked the Processors as running.");
// align items
AlignProcessGroupComponents alignProcessGroupComponents = new AlignProcessGroupComponents(restClient.getNiFiRestClient(), entity.getParentGroupId());
alignProcessGroupComponents.autoLayout();
}
templateCreationHelper.cleanupControllerServices();
log.info("Controller service cleanup complete");
List<NifiError> errors = templateCreationHelper.getErrors();
// add any global errors to the object
if (errors != null && !errors.isEmpty()) {
for (NifiError error : errors) {
newProcessGroup.addError(error);
}
}
newProcessGroup.setSuccess(!newProcessGroup.hasFatalErrors());
if (!newProcessGroup.isSuccess()) {
log.info("Errors while importing the template. {} errors found. {}", (errors != null ? errors.size() : 0), (errors != null ? " - " + StringUtils.join(errors, ",") : ""));
} else {
log.info("Success. Finished importing template ");
}
return newProcessGroup;
}
}
} catch (final Exception e) {
if (log.isDebugEnabled() && e instanceof WebApplicationException) {
final Response response = ((WebApplicationException) e).getResponse();
log.debug("NiFi server returned error: {}", response.readEntity(String.class), e);
}
throw new TemplateCreationException("Unable to create the template for the Id of [" + templateId + "]. " + e.getMessage(), e);
}
return null;
}
use of com.thinkbiganalytics.nifi.rest.model.NifiProperty in project kylo by Teradata.
the class LegacyNifiRestClient method updateProcessorProperties.
public void updateProcessorProperties(String processGroupId, String processorId, List<NifiProperty> properties) {
Map<String, NifiProperty> propertyMap = NifiPropertyUtil.propertiesAsMap(properties);
// fetch the processor
ProcessorDTO processor = getProcessor(processGroupId, processorId);
// iterate through and update the properties
for (Map.Entry<String, NifiProperty> property : propertyMap.entrySet()) {
processor.getConfig().getProperties().put(property.getKey(), property.getValue().getValue());
}
updateProcessor(processor);
}
use of com.thinkbiganalytics.nifi.rest.model.NifiProperty in project kylo by Teradata.
the class DefaultFeedManagerFeedService method createAndSaveFeed.
/**
* Create/Update a Feed in NiFi. Save the metadata to Kylo meta store.
*
* @param feedMetadata the feed metadata
* @return an object indicating if the feed creation was successful or not
*/
private NifiFeed createAndSaveFeed(FeedMetadata feedMetadata) {
Stopwatch stopwatch = Stopwatch.createStarted();
NifiFeed feed = null;
if (StringUtils.isBlank(feedMetadata.getId())) {
feedMetadata.setIsNew(true);
// If the feed is New we need to ensure the user has CREATE_FEED entity permission
if (accessController.isEntityAccessControlled()) {
metadataAccess.read(() -> {
// ensure the user has rights to create feeds under the category
Category domainCategory = categoryProvider.findById(categoryProvider.resolveId(feedMetadata.getCategory().getId()));
if (domainCategory == null) {
// throw exception
throw new MetadataRepositoryException("Unable to find the category " + feedMetadata.getCategory().getSystemName());
}
domainCategory.getAllowedActions().checkPermission(CategoryAccessControl.CREATE_FEED);
// ensure the user has rights to create feeds using the template
FeedManagerTemplate domainTemplate = templateProvider.findById(templateProvider.resolveId(feedMetadata.getTemplateId()));
if (domainTemplate == null) {
throw new MetadataRepositoryException("Unable to find the template " + feedMetadata.getTemplateId());
}
// domainTemplate.getAllowedActions().checkPermission(TemplateAccessControl.CREATE_FEED);
});
}
} else if (accessController.isEntityAccessControlled()) {
metadataAccess.read(() -> {
// perform explict entity access check here as we dont want to modify the NiFi flow unless user has access to edit the feed
Feed.ID domainId = feedProvider.resolveId(feedMetadata.getId());
Feed domainFeed = feedProvider.findById(domainId);
if (domainFeed != null) {
domainFeed.getAllowedActions().checkPermission(FeedAccessControl.EDIT_DETAILS);
} else {
throw new NotFoundException("Feed not found for id " + feedMetadata.getId());
}
});
}
// replace expressions with values
if (feedMetadata.getTable() != null) {
feedMetadata.getTable().updateMetadataFieldValues();
}
if (feedMetadata.getProperties() == null) {
feedMetadata.setProperties(new ArrayList<NifiProperty>());
}
// store ref to the originalFeedProperties before resolving and merging with the template
List<NifiProperty> originalFeedProperties = feedMetadata.getProperties();
// get all the properties for the metadata
RegisteredTemplate registeredTemplate = registeredTemplateService.findRegisteredTemplate(new RegisteredTemplateRequest.Builder().templateId(feedMetadata.getTemplateId()).templateName(feedMetadata.getTemplateName()).isFeedEdit(true).includeSensitiveProperties(true).build());
// copy the registered template properties it a new list so it doest get updated
List<NifiProperty> templateProperties = registeredTemplate.getProperties().stream().map(nifiProperty -> new NifiProperty(nifiProperty)).collect(Collectors.toList());
// update the template properties with the feedMetadata properties
List<NifiProperty> matchedProperties = NifiPropertyUtil.matchAndSetPropertyByProcessorName(templateProperties, feedMetadata.getProperties(), NifiPropertyUtil.PROPERTY_MATCH_AND_UPDATE_MODE.UPDATE_ALL_PROPERTIES);
registeredTemplate.setProperties(templateProperties);
feedMetadata.setProperties(registeredTemplate.getProperties());
feedMetadata.setRegisteredTemplate(registeredTemplate);
// skip any properties that the user supplied which are not ${ values
List<NifiProperty> propertiesToSkip = originalFeedProperties.stream().filter(property -> !propertyExpressionResolver.containsVariablesPatterns(property.getValue())).collect(Collectors.toList());
List<NifiProperty> templatePropertiesToSkip = registeredTemplate.getProperties().stream().filter(property -> property.isSelected() && !propertyExpressionResolver.containsVariablesPatterns(property.getValue())).collect(Collectors.toList());
if (templatePropertiesToSkip != null && !templatePropertiesToSkip.isEmpty()) {
propertiesToSkip.addAll(templatePropertiesToSkip);
}
// resolve any ${metadata.} properties
List<NifiProperty> resolvedProperties = propertyExpressionResolver.resolvePropertyExpressions(feedMetadata, propertiesToSkip);
// decrypt the metadata
feedModelTransform.decryptSensitivePropertyValues(feedMetadata);
FeedMetadata.STATE state = FeedMetadata.STATE.NEW;
try {
state = FeedMetadata.STATE.valueOf(feedMetadata.getState());
} catch (Exception e) {
// if the string isnt valid, disregard as it will end up disabling the feed.
}
boolean enabled = (FeedMetadata.STATE.NEW.equals(state) && feedMetadata.isActive()) || FeedMetadata.STATE.ENABLED.equals(state);
// flag to indicate to enable the feed later
// if this is the first time for this feed and it is set to be enabled, mark it to be enabled after we commit to the JCR store
boolean enableLater = false;
if (enabled && feedMetadata.isNew()) {
enableLater = true;
enabled = false;
feedMetadata.setState(FeedMetadata.STATE.DISABLED.name());
}
CreateFeedBuilder feedBuilder = CreateFeedBuilder.newFeed(nifiRestClient, nifiFlowCache, feedMetadata, registeredTemplate.getNifiTemplateId(), propertyExpressionResolver, propertyDescriptorTransform, niFiObjectCache, templateConnectionUtil).enabled(enabled).removeInactiveVersionedProcessGroup(removeInactiveNifiVersionedFeedFlows).autoAlign(nifiAutoFeedsAlignAfterSave).withNiFiTemplateCache(niFiTemplateCache);
if (registeredTemplate.isReusableTemplate()) {
feedBuilder.setReusableTemplate(true);
feedMetadata.setIsReusableFeed(true);
} else {
feedBuilder.inputProcessorType(feedMetadata.getInputProcessorType()).feedSchedule(feedMetadata.getSchedule()).properties(feedMetadata.getProperties());
if (registeredTemplate.usesReusableTemplate()) {
for (ReusableTemplateConnectionInfo connection : registeredTemplate.getReusableTemplateConnections()) {
feedBuilder.addInputOutputPort(new InputOutputPort(connection.getReusableTemplateInputPortName(), connection.getFeedOutputPortName()));
}
}
}
stopwatch.stop();
log.debug("Time to prepare data for saving feed in NiFi: {} ms", stopwatch.elapsed(TimeUnit.MILLISECONDS));
stopwatch.reset();
stopwatch.start();
NifiProcessGroup entity = feedBuilder.build();
stopwatch.stop();
log.debug("Time to save feed in NiFi: {} ms", stopwatch.elapsed(TimeUnit.MILLISECONDS));
stopwatch.reset();
feed = new NifiFeed(feedMetadata, entity);
// set the original feedProperties back to the feed
feedMetadata.setProperties(originalFeedProperties);
// encrypt the metadata properties
feedModelTransform.encryptSensitivePropertyValues(feedMetadata);
if (entity.isSuccess()) {
feedMetadata.setNifiProcessGroupId(entity.getProcessGroupEntity().getId());
try {
stopwatch.start();
saveFeed(feedMetadata);
// tell NiFi if this is a streaming feed or not
if (feedMetadata.getRegisteredTemplate().isStream()) {
streamingFeedJmsNotificationService.updateNiFiStatusJMSTopic(entity, feedMetadata);
}
feed.setEnableAfterSave(enableLater);
feed.setSuccess(true);
stopwatch.stop();
log.debug("Time to saveFeed in Kylo: {} ms", stopwatch.elapsed(TimeUnit.MILLISECONDS));
stopwatch.reset();
stopwatch.start();
feedBuilder.checkAndRemoveVersionedProcessGroup();
} catch (Exception e) {
feed.setSuccess(false);
feed.addErrorMessage(e);
}
} else {
feed.setSuccess(false);
}
if (!feed.isSuccess()) {
if (!entity.isRolledBack()) {
try {
feedBuilder.rollback();
} catch (FeedRollbackException rollbackException) {
log.error("Error rolling back feed {}. {} ", feedMetadata.getCategoryAndFeedName(), rollbackException.getMessage());
feed.addErrorMessage("Error occurred in rolling back the Feed.");
}
entity.setRolledBack(true);
}
}
return feed;
}
use of com.thinkbiganalytics.nifi.rest.model.NifiProperty in project kylo by Teradata.
the class DerivedDatasourceFactory method parseControllerServiceProperties.
/**
* Parse the defintion metadata for the {propertyKey:CS Property Key} objects and pick out the values in the controller service
*
* @param datasourceDefinition the definition to use
* @param feedProperties the feed properties that match this definition
* @return a Map of the Controller Service Property Key, Value
*/
private Map<String, String> parseControllerServiceProperties(DatasourceDefinition datasourceDefinition, List<NifiProperty> feedProperties) {
Map<String, String> properties = new HashMap<>();
try {
// {Source Database Connection:Database Connection URL}
List<String> controllerServiceProperties = datasourceDefinition.getDatasourcePropertyKeys().stream().filter(k -> k.matches("\\{(.*):(.*)\\}")).collect(Collectors.toList());
Map<String, List<String>> serviceProperties = new HashMap<>();
controllerServiceProperties.stream().forEach(p -> {
String service = p.substring(1, StringUtils.indexOf(p, ":"));
String property = p.substring(StringUtils.indexOf(p, ":") + 1, p.length() - 1);
if (!serviceProperties.containsKey(service)) {
serviceProperties.put(service, new ArrayList<>());
}
serviceProperties.get(service).add(property);
});
serviceProperties.entrySet().stream().forEach(e -> {
String service = e.getKey();
String controllerServiceId = feedProperties.stream().filter(p -> StringUtils.isNotBlank(p.getValue()) && p.getPropertyDescriptor() != null && p.getPropertyDescriptor().getName().equalsIgnoreCase(service) && StringUtils.isNotBlank(p.getPropertyDescriptor().getIdentifiesControllerService())).map(p -> p.getValue()).findFirst().orElse(null);
if (controllerServiceId != null) {
ControllerServiceDTO csDto = nifiControllerServiceProperties.getControllerServiceById(controllerServiceId);
if (csDto != null) {
e.getValue().stream().forEach(propertyKey -> {
String value = csDto.getProperties().get(propertyKey);
if (value != null) {
properties.put(propertyKey, value);
}
});
}
}
});
} catch (Exception e) {
log.warn("An error occurred trying to parse controller service properties when deriving the datasource for {}, {}. {} ", datasourceDefinition.getDatasourceType(), datasourceDefinition.getConnectionType(), e.getMessage(), e);
}
return properties;
}
use of com.thinkbiganalytics.nifi.rest.model.NifiProperty in project kylo by Teradata.
the class DerivedDatasourceFactory method ensureDatasource.
public Datasource.ID ensureDatasource(TemplateProcessorDatasourceDefinition definition, FeedMetadata feedMetadata, List<NifiProperty> allProperties) {
return metadataAccess.commit(() -> {
List<NifiProperty> propertiesToEvalulate = new ArrayList<NifiProperty>();
// fetch the def
DatasourceDefinition datasourceDefinition = datasourceDefinitionProvider.findByProcessorType(definition.getProcessorType());
if (datasourceDefinition != null) {
// find out if there are any saved properties on the Feed that match the datasourceDef
List<NifiProperty> feedProperties = feedMetadata.getProperties().stream().filter(property -> matchesDefinition(definition, property) && datasourceDefinition.getDatasourcePropertyKeys().contains(property.getKey())).collect(Collectors.toList());
// resolve any ${metadata.} properties
List<NifiProperty> resolvedFeedProperties = propertyExpressionResolver.resolvePropertyExpressions(feedProperties, feedMetadata);
List<NifiProperty> resolvedAllProperties = propertyExpressionResolver.resolvePropertyExpressions(allProperties, feedMetadata);
// propetyHash
propertiesToEvalulate.addAll(feedProperties);
propertiesToEvalulate.addAll(allProperties);
propertyExpressionResolver.resolveStaticProperties(propertiesToEvalulate);
String identityString = datasourceDefinition.getIdentityString();
String desc = datasourceDefinition.getDescription();
String title = datasourceDefinition.getTitle();
PropertyExpressionResolver.ResolvedVariables identityStringPropertyResolution = propertyExpressionResolver.resolveVariables(identityString, propertiesToEvalulate);
identityString = identityStringPropertyResolution.getResolvedString();
PropertyExpressionResolver.ResolvedVariables titlePropertyResolution = propertyExpressionResolver.resolveVariables(title, propertiesToEvalulate);
title = titlePropertyResolution.getResolvedString();
if (desc != null) {
PropertyExpressionResolver.ResolvedVariables descriptionPropertyResolution = propertyExpressionResolver.resolveVariables(desc, propertiesToEvalulate);
desc = descriptionPropertyResolution.getResolvedString();
}
// if the identityString still contains unresolved variables then make the title readable and replace the idstring with the feed.id
if (propertyExpressionResolver.containsVariablesPatterns(identityString)) {
title = propertyExpressionResolver.replaceAll(title, " {runtime variable} ");
identityString = propertyExpressionResolver.replaceAll(identityString, feedMetadata.getId());
}
// if it is the Source ensure the feed matches this ds
if (isCreateDatasource(datasourceDefinition, feedMetadata)) {
Map<String, String> controllerServiceProperties = parseControllerServiceProperties(datasourceDefinition, feedProperties);
Map<String, Object> properties = new HashMap<String, Object>(identityStringPropertyResolution.getResolvedVariables());
properties.putAll(controllerServiceProperties);
DerivedDatasource derivedDatasource = datasourceProvider.ensureDerivedDatasource(datasourceDefinition.getDatasourceType(), identityString, title, desc, properties);
if (derivedDatasource != null) {
if ("HiveDatasource".equals(derivedDatasource.getDatasourceType()) && Optional.ofNullable(feedMetadata.getTable()).map(TableSetup::getTableSchema).map(TableSchema::getFields).isPresent()) {
derivedDatasource.setGenericProperties(Collections.singletonMap("columns", (Serializable) feedMetadata.getTable().getTableSchema().getFields()));
}
return derivedDatasource.getId();
}
}
return null;
} else {
return null;
}
}, MetadataAccess.SERVICE);
}
Aggregations