Examples with DatasourceDefinition com.thinkbiganalytics.metadata.api.datasource.DatasourceDefinition used on opensource projects

Example 1 with DatasourceDefinition

use of com.thinkbiganalytics.metadata.api.datasource.DatasourceDefinition in project kylo by Teradata.

the class JcrDatasourceDefinitionProvider method ensureDatasourceDefinition.

public DatasourceDefinition ensureDatasourceDefinition(String processorType) {
    DatasourceDefinition dsDef = findByProcessorType(processorType);
    if (dsDef == null) {
        try {
            if (!getSession().getRootNode().hasNode("metadata/datasourceDefinitions")) {
                getSession().getRootNode().addNode("metadata/datasourceDefinitions", "tba:datasourceDefinitionsFolder");
            }
        } catch (RepositoryException e) {
            log.error("Failed to create datasource definitions node", e);
        }
        String path = EntityUtil.pathForDatasourceDefinition();
        Node node = findOrCreateEntityNode(path, processorType, getJcrEntityClass());
        JcrDatasourceDefinition def = new JcrDatasourceDefinition(node);
        dsDef = def;
    }
    return dsDef;
}

Example 2 with DatasourceDefinition

use of com.thinkbiganalytics.metadata.api.datasource.DatasourceDefinition in project kylo by Teradata.

the class DerivedDatasourceFactory method ensureDataTransformationSourceDatasources.

/**
 * Builds the list of data sources for the specified data transformation feed.
 *
 * @param feed the feed
 * @return the list of data sources
 * @throws NullPointerException if the feed has no data transformation
 */
@Nonnull
private Set<Datasource.ID> ensureDataTransformationSourceDatasources(@Nonnull final FeedMetadata feed) {
    final Set<Datasource.ID> datasources = new HashSet<>();
    // Extract nodes in chart view model
    @SuppressWarnings("unchecked") final Stream<Map<String, Object>> nodes = Optional.ofNullable(feed.getDataTransformation().getChartViewModel()).map(model -> (List<Map<String, Object>>) model.get("nodes")).map(Collection::stream).orElse(Stream.empty());
    // Create a data source for each node
    final DatasourceDefinition hiveDefinition = datasourceDefinitionProvider.findByProcessorType(DATA_TRANSFORMATION_HIVE_DEFINITION);
    final DatasourceDefinition jdbcDefinition = datasourceDefinitionProvider.findByProcessorType(DATA_TRANSFORMATION_JDBC_DEFINITION);
    nodes.forEach(node -> {
        // Extract properties from node
        final DatasourceDefinition datasourceDefinition;
        final Map<String, String> properties = new HashMap<>();
        if (node.get("datasourceId") == null || node.get("datasourceId").equals("HIVE")) {
            final String name = (String) node.get("name");
            datasourceDefinition = hiveDefinition;
            properties.put(HIVE_SCHEMA_KEY, StringUtils.trim(StringUtils.substringBefore(name, ".")));
            properties.put(HIVE_TABLE_KEY, StringUtils.trim(StringUtils.substringAfterLast(name, ".")));
        } else {
            final Datasource datasource = datasourceProvider.getDatasource(datasourceProvider.resolve((String) node.get("datasourceId")));
            datasourceDefinition = jdbcDefinition;
            properties.put(JDBC_CONNECTION_KEY, datasource.getName());
            properties.put(JDBC_TABLE_KEY, (String) node.get("name"));
            properties.putAll(parseDataTransformControllerServiceProperties(datasourceDefinition, datasource.getName()));
        }
        if (datasourceDefinition != null) {
            // Create the derived data source
            final String identityString = propertyExpressionResolver.resolveVariables(datasourceDefinition.getIdentityString(), properties);
            final String title = datasourceDefinition.getTitle() != null ? propertyExpressionResolver.resolveVariables(datasourceDefinition.getTitle(), properties) : identityString;
            final String desc = propertyExpressionResolver.resolveVariables(datasourceDefinition.getDescription(), properties);
            final DerivedDatasource datasource = datasourceProvider.ensureDerivedDatasource(datasourceDefinition.getDatasourceType(), identityString, title, desc, new HashMap<>(properties));
            datasources.add(datasource.getId());
        }
    });
    // Build the data sources from the data source ids
    final List<String> datasourceIds = Optional.ofNullable(feed.getDataTransformation()).map(FeedDataTransformation::getDatasourceIds).orElse(Collections.emptyList());
    datasourceIds.stream().map(datasourceProvider::resolve).forEach(datasources::add);
    return datasources;
}

Example 3 with DatasourceDefinition

use of com.thinkbiganalytics.metadata.api.datasource.DatasourceDefinition in project kylo by Teradata.

the class DerivedDatasourceFactory method parseDataTransformControllerServiceProperties.

private Map<String, String> parseDataTransformControllerServiceProperties(DatasourceDefinition datasourceDefinition, String controllerServiceName) {
    Map<String, String> properties = new HashMap<>();
    if (datasourceDefinition != null) {
        try {
            if (StringUtils.isNotBlank(controllerServiceName)) {
                // {Source Database Connection:Database Connection URL}
                List<String> controllerServiceProperties = datasourceDefinition.getDatasourcePropertyKeys().stream().filter(k -> k.matches("\\{" + JDBC_CONNECTION_KEY + ":(.*)\\}")).collect(Collectors.toList());
                List<String> serviceProperties = new ArrayList<>();
                controllerServiceProperties.stream().forEach(p -> {
                    String property = p.substring(StringUtils.indexOf(p, ":") + 1, p.length() - 1);
                    serviceProperties.add(property);
                });
                ControllerServiceDTO csDto = nifiControllerServiceProperties.getControllerServiceByName(controllerServiceName);
                if (csDto != null) {
                    serviceProperties.stream().forEach(p -> {
                        if (csDto != null) {
                            String value = csDto.getProperties().get(p);
                            if (value != null) {
                                properties.put(p, value);
                            }
                        }
                    });
                }
            }
        } catch (Exception e) {
            log.warn("An error occurred trying to parse controller service properties for data transformation when deriving the datasource for {}, {}. {} ", datasourceDefinition.getDatasourceType(), datasourceDefinition.getConnectionType(), e.getMessage(), e);
        }
    }
    return properties;
}

Example 4 with DatasourceDefinition

use of com.thinkbiganalytics.metadata.api.datasource.DatasourceDefinition in project kylo by Teradata.

the class JcrDerivedDatasource method setDatasourceDefinitions.

public void setDatasourceDefinitions(Set<DatasourceDefinition> datasourceDefinitions) {
    JcrPropertyUtil.setProperty(this.node, DATASOURCE_DEFINITION, null);
    for (DatasourceDefinition dest : datasourceDefinitions) {
        Node destNode = ((JcrDatasourceDefinition) dest).getNode();
        addDatasourceDefinition(destNode);
    }
}

Example 5 with DatasourceDefinition

use of com.thinkbiganalytics.metadata.api.datasource.DatasourceDefinition in project kylo by Teradata.

the class DerivedDatasourceFactory method parseControllerServiceProperties.

/**
 * Parse the defintion metadata for the {propertyKey:CS Property Key} objects and pick out the values in the controller service
 *
 * @param datasourceDefinition the definition to use
 * @param feedProperties       the feed properties that match this definition
 * @return a Map of the Controller Service Property Key, Value
 */
private Map<String, String> parseControllerServiceProperties(DatasourceDefinition datasourceDefinition, List<NifiProperty> feedProperties) {
    Map<String, String> properties = new HashMap<>();
    try {
        // {Source Database Connection:Database Connection URL}
        List<String> controllerServiceProperties = datasourceDefinition.getDatasourcePropertyKeys().stream().filter(k -> k.matches("\\{(.*):(.*)\\}")).collect(Collectors.toList());
        Map<String, List<String>> serviceProperties = new HashMap<>();
        controllerServiceProperties.stream().forEach(p -> {
            String service = p.substring(1, StringUtils.indexOf(p, ":"));
            String property = p.substring(StringUtils.indexOf(p, ":") + 1, p.length() - 1);
            if (!serviceProperties.containsKey(service)) {
                serviceProperties.put(service, new ArrayList<>());
            }
            serviceProperties.get(service).add(property);
        });
        serviceProperties.entrySet().stream().forEach(e -> {
            String service = e.getKey();
            String controllerServiceId = feedProperties.stream().filter(p -> StringUtils.isNotBlank(p.getValue()) && p.getPropertyDescriptor() != null && p.getPropertyDescriptor().getName().equalsIgnoreCase(service) && StringUtils.isNotBlank(p.getPropertyDescriptor().getIdentifiesControllerService())).map(p -> p.getValue()).findFirst().orElse(null);
            if (controllerServiceId != null) {
                ControllerServiceDTO csDto = nifiControllerServiceProperties.getControllerServiceById(controllerServiceId);
                if (csDto != null) {
                    e.getValue().stream().forEach(propertyKey -> {
                        String value = csDto.getProperties().get(propertyKey);
                        if (value != null) {
                            properties.put(propertyKey, value);
                        }
                    });
                }
            }
        });
    } catch (Exception e) {
        log.warn("An error occurred trying to parse controller service properties when deriving the datasource for {}, {}. {} ", datasourceDefinition.getDatasourceType(), datasourceDefinition.getConnectionType(), e.getMessage(), e);
    }
    return properties;
}