Examples with TemplateProcessorDatasourceDefinition com.thinkbiganalytics.feedmgr.rest.model.TemplateProcessorDatasourceDefinition used on opensource projects

Example 1 with TemplateProcessorDatasourceDefinition

use of com.thinkbiganalytics.feedmgr.rest.model.TemplateProcessorDatasourceDefinition in project kylo by Teradata.

the class TemplatesRestController method getNiFiTemplateFlowInfo.

/**
 * Returns data about the NiFiTemplate and its processors related to the input connections, along with the Datasources in the flow
 */
@POST
@Path("/nifi/{templateId}/flow-info")
@Consumes(MediaType.APPLICATION_JSON)
@Produces(MediaType.APPLICATION_JSON)
@ApiOperation("Gets the flow for the specified template.")
@ApiResponses({ @ApiResponse(code = 200, message = "Returns the flow.", response = NiFiTemplateFlowResponse.class), @ApiResponse(code = 500, message = "NiFi is unavailable.", response = RestResponseStatus.class) })
public Response getNiFiTemplateFlowInfo(@PathParam("templateId") String templateId, NiFiTemplateFlowRequest flowRequest) {
    this.accessController.checkPermission(AccessController.SERVICES, FeedServicesAccessControl.ACCESS_TEMPLATES);
    List<TemplateProcessorDatasourceDefinition> templateProcessorDatasourceDefinitions = new ArrayList<>();
    NiFiTemplateFlowResponse response = new NiFiTemplateFlowResponse();
    response.setRequest(flowRequest);
    if (StringUtils.isNotBlank(templateId)) {
        List<RegisteredTemplate.FlowProcessor> processors = feedManagerTemplateService.getNiFiTemplateFlowProcessors(templateId, flowRequest.getConnectionInfo());
        Set<DatasourceDefinition> defs = datasourceService.getDatasourceDefinitions();
        Map<String, DatasourceDefinition> datasourceDefinitionMap = new HashMap<>();
        if (defs != null) {
            defs.stream().forEach(def -> datasourceDefinitionMap.put(def.getProcessorType(), def));
        }
        templateProcessorDatasourceDefinitions = processors.stream().filter(processor -> datasourceDefinitionMap.containsKey(processor.getType())).map(p -> {
            TemplateProcessorDatasourceDefinition definition = new TemplateProcessorDatasourceDefinition();
            definition.setProcessorType(p.getType());
            definition.setProcessorName(p.getName());
            definition.setProcessorId(p.getId());
            definition.setDatasourceDefinition(datasourceDefinitionMap.get(p.getType()));
            return definition;
        }).collect(Collectors.toList());
        response.setProcessors(processors);
        response.setTemplateProcessorDatasourceDefinitions(templateProcessorDatasourceDefinitions);
    }
    return Response.ok(response).build();
}

Example 2 with TemplateProcessorDatasourceDefinition

use of com.thinkbiganalytics.feedmgr.rest.model.TemplateProcessorDatasourceDefinition in project kylo by Teradata.

the class DerivedDatasourceFactory method ensureDatasource.

public Datasource.ID ensureDatasource(TemplateProcessorDatasourceDefinition definition, FeedMetadata feedMetadata, List<NifiProperty> allProperties) {
    return metadataAccess.commit(() -> {
        List<NifiProperty> propertiesToEvalulate = new ArrayList<NifiProperty>();
        // fetch the def
        DatasourceDefinition datasourceDefinition = datasourceDefinitionProvider.findByProcessorType(definition.getProcessorType());
        if (datasourceDefinition != null) {
            // find out if there are any saved properties on the Feed that match the datasourceDef
            List<NifiProperty> feedProperties = feedMetadata.getProperties().stream().filter(property -> matchesDefinition(definition, property) && datasourceDefinition.getDatasourcePropertyKeys().contains(property.getKey())).collect(Collectors.toList());
            // resolve any ${metadata.} properties
            List<NifiProperty> resolvedFeedProperties = propertyExpressionResolver.resolvePropertyExpressions(feedProperties, feedMetadata);
            List<NifiProperty> resolvedAllProperties = propertyExpressionResolver.resolvePropertyExpressions(allProperties, feedMetadata);
            // propetyHash
            propertiesToEvalulate.addAll(feedProperties);
            propertiesToEvalulate.addAll(allProperties);
            propertyExpressionResolver.resolveStaticProperties(propertiesToEvalulate);
            String identityString = datasourceDefinition.getIdentityString();
            String desc = datasourceDefinition.getDescription();
            String title = datasourceDefinition.getTitle();
            PropertyExpressionResolver.ResolvedVariables identityStringPropertyResolution = propertyExpressionResolver.resolveVariables(identityString, propertiesToEvalulate);
            identityString = identityStringPropertyResolution.getResolvedString();
            PropertyExpressionResolver.ResolvedVariables titlePropertyResolution = propertyExpressionResolver.resolveVariables(title, propertiesToEvalulate);
            title = titlePropertyResolution.getResolvedString();
            if (desc != null) {
                PropertyExpressionResolver.ResolvedVariables descriptionPropertyResolution = propertyExpressionResolver.resolveVariables(desc, propertiesToEvalulate);
                desc = descriptionPropertyResolution.getResolvedString();
            }
            // if the identityString still contains unresolved variables then make the title readable and replace the idstring with the feed.id
            if (propertyExpressionResolver.containsVariablesPatterns(identityString)) {
                title = propertyExpressionResolver.replaceAll(title, " {runtime variable} ");
                identityString = propertyExpressionResolver.replaceAll(identityString, feedMetadata.getId());
            }
            // if it is the Source ensure the feed matches this ds
            if (isCreateDatasource(datasourceDefinition, feedMetadata)) {
                Map<String, String> controllerServiceProperties = parseControllerServiceProperties(datasourceDefinition, feedProperties);
                Map<String, Object> properties = new HashMap<String, Object>(identityStringPropertyResolution.getResolvedVariables());
                properties.putAll(controllerServiceProperties);
                DerivedDatasource derivedDatasource = datasourceProvider.ensureDerivedDatasource(datasourceDefinition.getDatasourceType(), identityString, title, desc, properties);
                if (derivedDatasource != null) {
                    if ("HiveDatasource".equals(derivedDatasource.getDatasourceType()) && Optional.ofNullable(feedMetadata.getTable()).map(TableSetup::getTableSchema).map(TableSchema::getFields).isPresent()) {
                        derivedDatasource.setGenericProperties(Collections.singletonMap("columns", (Serializable) feedMetadata.getTable().getTableSchema().getFields()));
                    }
                    return derivedDatasource.getId();
                }
            }
            return null;
        } else {
            return null;
        }
    }, MetadataAccess.SERVICE);
}