Examples with DerivedDatasource com.thinkbiganalytics.metadata.api.datasource.DerivedDatasource used on opensource projects

Example 1 with DerivedDatasource

use of com.thinkbiganalytics.metadata.api.datasource.DerivedDatasource in project kylo by Teradata.

the class HiveColumnsUpgradeAction method upgradeTo.

@Override
public void upgradeTo(final KyloVersion startingVersion) {
    log.info("Upgrading hive columns from version: {}", startingVersion);
    feedService.getFeeds().stream().filter(feed -> Optional.ofNullable(feed.getTable()).map(TableSetup::getTableSchema).map(TableSchema::getFields).isPresent()).forEach(feed -> {
        final TableSchema schema = feed.getTable().getTableSchema();
        final DerivedDatasource datasource = datasourceProvider.findDerivedDatasource("HiveDatasource", feed.getSystemCategoryName() + "." + feed.getSystemFeedName());
        if (datasource != null) {
            log.info("Upgrading schema: {}/{}", schema.getDatabaseName(), schema.getSchemaName());
            datasource.setGenericProperties(Collections.singletonMap("columns", (Serializable) schema.getFields()));
        }
    });
}

Example 2 with DerivedDatasource

use of com.thinkbiganalytics.metadata.api.datasource.DerivedDatasource in project kylo by Teradata.

the class FeedManagerFeedTest method testFeedDatasource.

@Test
public void testFeedDatasource() {
    String categorySystemName = "my_category";
    String feedName = "my_feed";
    String templateName = "my_template";
    String description = " my feed description";
    setupFeedAndTemplate(categorySystemName, feedName, templateName);
    // boolean isDefineTable = true;
    // boolean isGetFile = false;
    metadata.commit(() -> {
        Feed feed = feedTestUtil.findFeed(categorySystemName, feedName);
        Set<Datasource.ID> sources = new HashSet<Datasource.ID>();
        Set<com.thinkbiganalytics.metadata.api.datasource.Datasource.ID> destinations = new HashSet<>();
        // Add Table Dependencies
        String uniqueName = FeedNameUtil.fullName(categorySystemName, feedName);
        DerivedDatasource srcDatasource = datasourceProvider.ensureDatasource(uniqueName, feed.getDescription(), DerivedDatasource.class);
        sources.add(srcDatasource.getId());
        DerivedDatasource destDatasource = datasourceProvider.ensureDatasource("destination", feed.getDescription(), DerivedDatasource.class);
        destinations.add(destDatasource.getId());
        sources.stream().forEach(sourceId -> feedProvider.ensureFeedSource(feed.getId(), sourceId));
        destinations.stream().forEach(destinationId -> feedProvider.ensureFeedDestination(feed.getId(), destinationId));
    }, MetadataAccess.SERVICE);
    // ensure the sources and dest got created
    metadata.read(() -> {
        Feed feed = feedTestUtil.findFeed(categorySystemName, feedName);
        Assert.assertNotNull(feed.getSources());
        Assert.assertTrue(feed.getSources().size() == 1, "Feed Sources should be 1");
        Assert.assertNotNull(feed.getDestinations());
        Assert.assertTrue(feed.getDestinations().size() == 1, "Feed Destinations should be 1");
        List<? extends FeedDestination> feedDestinations = feed.getDestinations();
        if (feedDestinations != null) {
            FeedDestination feedDestination = feedDestinations.get(0);
            Datasource ds = feedDestination.getDatasource();
            Assert.assertTrue(ds instanceof DerivedDatasource, "Datasource was not expected DerivedDatasource");
        }
    }, MetadataAccess.SERVICE);
}

Example 3 with DerivedDatasource

use of com.thinkbiganalytics.metadata.api.datasource.DerivedDatasource in project kylo by Teradata.

the class DerivedDatasourceFactory method ensureDataTransformationSourceDatasources.

/**
 * Builds the list of data sources for the specified data transformation feed.
 *
 * @param feed the feed
 * @return the list of data sources
 * @throws NullPointerException if the feed has no data transformation
 */
@Nonnull
private Set<Datasource.ID> ensureDataTransformationSourceDatasources(@Nonnull final FeedMetadata feed) {
    final Set<Datasource.ID> datasources = new HashSet<>();
    // Extract nodes in chart view model
    @SuppressWarnings("unchecked") final Stream<Map<String, Object>> nodes = Optional.ofNullable(feed.getDataTransformation().getChartViewModel()).map(model -> (List<Map<String, Object>>) model.get("nodes")).map(Collection::stream).orElse(Stream.empty());
    // Create a data source for each node
    final DatasourceDefinition hiveDefinition = datasourceDefinitionProvider.findByProcessorType(DATA_TRANSFORMATION_HIVE_DEFINITION);
    final DatasourceDefinition jdbcDefinition = datasourceDefinitionProvider.findByProcessorType(DATA_TRANSFORMATION_JDBC_DEFINITION);
    nodes.forEach(node -> {
        // Extract properties from node
        final DatasourceDefinition datasourceDefinition;
        final Map<String, String> properties = new HashMap<>();
        if (node.get("datasourceId") == null || node.get("datasourceId").equals("HIVE")) {
            final String name = (String) node.get("name");
            datasourceDefinition = hiveDefinition;
            properties.put(HIVE_SCHEMA_KEY, StringUtils.trim(StringUtils.substringBefore(name, ".")));
            properties.put(HIVE_TABLE_KEY, StringUtils.trim(StringUtils.substringAfterLast(name, ".")));
        } else {
            final Datasource datasource = datasourceProvider.getDatasource(datasourceProvider.resolve((String) node.get("datasourceId")));
            datasourceDefinition = jdbcDefinition;
            properties.put(JDBC_CONNECTION_KEY, datasource.getName());
            properties.put(JDBC_TABLE_KEY, (String) node.get("name"));
            properties.putAll(parseDataTransformControllerServiceProperties(datasourceDefinition, datasource.getName()));
        }
        if (datasourceDefinition != null) {
            // Create the derived data source
            final String identityString = propertyExpressionResolver.resolveVariables(datasourceDefinition.getIdentityString(), properties);
            final String title = datasourceDefinition.getTitle() != null ? propertyExpressionResolver.resolveVariables(datasourceDefinition.getTitle(), properties) : identityString;
            final String desc = propertyExpressionResolver.resolveVariables(datasourceDefinition.getDescription(), properties);
            final DerivedDatasource datasource = datasourceProvider.ensureDerivedDatasource(datasourceDefinition.getDatasourceType(), identityString, title, desc, new HashMap<>(properties));
            datasources.add(datasource.getId());
        }
    });
    // Build the data sources from the data source ids
    final List<String> datasourceIds = Optional.ofNullable(feed.getDataTransformation()).map(FeedDataTransformation::getDatasourceIds).orElse(Collections.emptyList());
    datasourceIds.stream().map(datasourceProvider::resolve).forEach(datasources::add);
    return datasources;
}

Example 4 with DerivedDatasource

use of com.thinkbiganalytics.metadata.api.datasource.DerivedDatasource in project kylo by Teradata.

the class JcrDatasourceProvider method ensureDerivedDatasource.

/**
 * gets or creates the Derived datasource
 */
public DerivedDatasource ensureDerivedDatasource(String datasourceType, String identityString, String title, String desc, Map<String, Object> properties) {
    // ensure the identity String is not null
    if (identityString == null) {
        identityString = "";
    }
    if (datasourceType == null) {
        datasourceType = "Datasource";
    }
    DerivedDatasource derivedDatasource = findDerivedDatasource(datasourceType, identityString);
    if (derivedDatasource == null) {
        try {
            if (!getSession().getRootNode().hasNode("metadata/datasources/derived")) {
                if (!getSession().getRootNode().hasNode("metadata/datasources")) {
                    getSession().getRootNode().addNode("metadata", "datasources");
                }
                getSession().getRootNode().getNode("metadata/datasources").addNode("derived");
            }
            Node parentNode = getSession().getNode(EntityUtil.pathForDerivedDatasource());
            String nodeName = datasourceType + "-" + identityString;
            if (Jsr283Encoder.containsEncodeableCharacters(identityString)) {
                nodeName = new Jsr283Encoder().encode(nodeName);
            }
            JcrDerivedDatasource jcrDerivedDatasource = null;
            try {
                jcrDerivedDatasource = findDerivedDatasourceByNodeName(nodeName);
            } catch (RepositoryException e) {
                log.warn("An exception occurred trying to find the DerivedDatasource by node name {}.  {} ", nodeName, e.getMessage());
            }
            derivedDatasource = jcrDerivedDatasource;
            if (jcrDerivedDatasource == null) {
                Node derivedDatasourceNode = JcrUtil.createNode(parentNode, nodeName, JcrDerivedDatasource.NODE_TYPE);
                jcrDerivedDatasource = new JcrDerivedDatasource(derivedDatasourceNode);
                jcrDerivedDatasource.setSystemName(identityString);
                jcrDerivedDatasource.setDatasourceType(datasourceType);
                jcrDerivedDatasource.setTitle(title);
                jcrDerivedDatasource.setDescription(desc);
                derivedDatasource = jcrDerivedDatasource;
            }
        } catch (RepositoryException e) {
            log.error("Failed to create Derived Datasource for DatasourceType: {}, IdentityString: {}, Error: {}", datasourceType, identityString, e.getMessage(), e);
        }
    }
    if (derivedDatasource != null) {
        // ((JcrDerivedDatasource)derivedDatasource).mergeProperties()
        if (properties != null) {
            derivedDatasource.setProperties(properties);
        }
        derivedDatasource.setTitle(title);
    }
    return derivedDatasource;
}

Example 5 with DerivedDatasource

use of com.thinkbiganalytics.metadata.api.datasource.DerivedDatasource in project kylo by Teradata.

the class DerivedDatasourceFactory method ensureDatasource.

public Datasource.ID ensureDatasource(TemplateProcessorDatasourceDefinition definition, FeedMetadata feedMetadata, List<NifiProperty> allProperties) {
    return metadataAccess.commit(() -> {
        List<NifiProperty> propertiesToEvalulate = new ArrayList<NifiProperty>();
        // fetch the def
        DatasourceDefinition datasourceDefinition = datasourceDefinitionProvider.findByProcessorType(definition.getProcessorType());
        if (datasourceDefinition != null) {
            // find out if there are any saved properties on the Feed that match the datasourceDef
            List<NifiProperty> feedProperties = feedMetadata.getProperties().stream().filter(property -> matchesDefinition(definition, property) && datasourceDefinition.getDatasourcePropertyKeys().contains(property.getKey())).collect(Collectors.toList());
            // resolve any ${metadata.} properties
            List<NifiProperty> resolvedFeedProperties = propertyExpressionResolver.resolvePropertyExpressions(feedProperties, feedMetadata);
            List<NifiProperty> resolvedAllProperties = propertyExpressionResolver.resolvePropertyExpressions(allProperties, feedMetadata);
            // propetyHash
            propertiesToEvalulate.addAll(feedProperties);
            propertiesToEvalulate.addAll(allProperties);
            propertyExpressionResolver.resolveStaticProperties(propertiesToEvalulate);
            String identityString = datasourceDefinition.getIdentityString();
            String desc = datasourceDefinition.getDescription();
            String title = datasourceDefinition.getTitle();
            PropertyExpressionResolver.ResolvedVariables identityStringPropertyResolution = propertyExpressionResolver.resolveVariables(identityString, propertiesToEvalulate);
            identityString = identityStringPropertyResolution.getResolvedString();
            PropertyExpressionResolver.ResolvedVariables titlePropertyResolution = propertyExpressionResolver.resolveVariables(title, propertiesToEvalulate);
            title = titlePropertyResolution.getResolvedString();
            if (desc != null) {
                PropertyExpressionResolver.ResolvedVariables descriptionPropertyResolution = propertyExpressionResolver.resolveVariables(desc, propertiesToEvalulate);
                desc = descriptionPropertyResolution.getResolvedString();
            }
            // if the identityString still contains unresolved variables then make the title readable and replace the idstring with the feed.id
            if (propertyExpressionResolver.containsVariablesPatterns(identityString)) {
                title = propertyExpressionResolver.replaceAll(title, " {runtime variable} ");
                identityString = propertyExpressionResolver.replaceAll(identityString, feedMetadata.getId());
            }
            // if it is the Source ensure the feed matches this ds
            if (isCreateDatasource(datasourceDefinition, feedMetadata)) {
                Map<String, String> controllerServiceProperties = parseControllerServiceProperties(datasourceDefinition, feedProperties);
                Map<String, Object> properties = new HashMap<String, Object>(identityStringPropertyResolution.getResolvedVariables());
                properties.putAll(controllerServiceProperties);
                DerivedDatasource derivedDatasource = datasourceProvider.ensureDerivedDatasource(datasourceDefinition.getDatasourceType(), identityString, title, desc, properties);
                if (derivedDatasource != null) {
                    if ("HiveDatasource".equals(derivedDatasource.getDatasourceType()) && Optional.ofNullable(feedMetadata.getTable()).map(TableSetup::getTableSchema).map(TableSchema::getFields).isPresent()) {
                        derivedDatasource.setGenericProperties(Collections.singletonMap("columns", (Serializable) feedMetadata.getTable().getTableSchema().getFields()));
                    }
                    return derivedDatasource.getId();
                }
            }
            return null;
        } else {
            return null;
        }
    }, MetadataAccess.SERVICE);
}