Examples with DerivedDatasource com.thinkbiganalytics.metadata.api.datasource.DerivedDatasource used on opensource projects

Example 6 with DerivedDatasource

use of com.thinkbiganalytics.metadata.api.datasource.DerivedDatasource in project kylo by Teradata.

the class DerivedDatasourceFactory method ensureDataTransformationDestinationDatasources.

/**
 * Builds the list of destinations for the specified data transformation feed.
 *
 * <p>The data source type is determined based on the sources used in the transformation. If only one source is used then it is assumed that the source and destination are the same. Otherwise it
 * is assumed that the destination is Hive.</p>
 *
 * @param feed the feed
 * @return the list of destinations
 * @throws NullPointerException if the feed has no data transformation
 */
@Nonnull
private Set<Datasource.ID> ensureDataTransformationDestinationDatasources(@Nonnull final FeedMetadata feed) {
    // Set properties based on data source type
    final String processorType;
    final Map<String, String> properties = new HashMap<>();
    if (feed.getDataTransformation().getDatasourceIds() != null && feed.getDataTransformation().getDatasourceIds().size() == 1) {
        final Datasource datasource = datasourceProvider.getDatasource(datasourceProvider.resolve(feed.getDataTransformation().getDatasourceIds().get(0)));
        processorType = DATA_TRANSFORMATION_JDBC_DEFINITION;
        properties.put(JDBC_CONNECTION_KEY, datasource.getName());
        properties.put(JDBC_TABLE_KEY, feed.getSystemCategoryName() + "." + feed.getSystemFeedName());
    } else {
        processorType = DATA_TRANSFORMATION_HIVE_DEFINITION;
        properties.put(HIVE_SCHEMA_KEY, feed.getSystemCategoryName());
        properties.put(HIVE_TABLE_KEY, feed.getSystemFeedName());
    }
    // Create datasource
    final DatasourceDefinition datasourceDefinition = datasourceDefinitionProvider.findByProcessorType(processorType);
    if (datasourceDefinition != null) {
        final String identityString = propertyExpressionResolver.resolveVariables(datasourceDefinition.getIdentityString(), properties);
        final String title = datasourceDefinition.getTitle() != null ? propertyExpressionResolver.resolveVariables(datasourceDefinition.getTitle(), properties) : identityString;
        final String desc = propertyExpressionResolver.resolveVariables(datasourceDefinition.getDescription(), properties);
        if (processorType.equals(DATA_TRANSFORMATION_JDBC_DEFINITION)) {
            properties.putAll(parseDataTransformControllerServiceProperties(datasourceDefinition, properties.get(JDBC_CONNECTION_KEY)));
        }
        final DerivedDatasource datasource = datasourceProvider.ensureDerivedDatasource(datasourceDefinition.getDatasourceType(), identityString, title, desc, new HashMap<>(properties));
        return Collections.singleton(datasource.getId());
    } else {
        return Collections.emptySet();
    }
}

Example 7 with DerivedDatasource

use of com.thinkbiganalytics.metadata.api.datasource.DerivedDatasource in project kylo by Teradata.

the class JcrIndexService method indexDerivedDatasource.

/**
 * Indexes derived datasource objects.
 *
 * @param datasource the derived datasource to index
 * @return {@code true} if the index was updated, or {@code false} otherwise
 */
private boolean indexDerivedDatasource(@Nonnull final DerivedDatasource datasource) {
    if (HIVE_DATASOURCE.equals(datasource.getDatasourceType())) {
        boolean allowIndexing = metadataAccess.read(() -> {
            boolean allowIndexingEvaluation = true;
            Session session = JcrMetadataAccess.getActiveSession();
            if (session != null) {
                Value[] feedDestinationsArray = ((JcrDerivedDatasource) datasource).getNode().getProperty("tba:feedDestinations").getValues();
                for (Value feedDestination : feedDestinationsArray) {
                    Node feedDestinationNode = session.getNodeByIdentifier(feedDestination.getString());
                    if (feedDestinationNode != null) {
                        Node feedDetailsNode = feedDestinationNode.getParent();
                        if (feedDetailsNode != null) {
                            Node feedSummaryNode = feedDetailsNode.getParent();
                            if (feedSummaryNode != null) {
                                String indexingOption = feedSummaryNode.getProperty("tba:allowIndexing").getString();
                                if ((indexingOption != null) && (indexingOption.equals("N"))) {
                                    allowIndexingEvaluation = false;
                                }
                            }
                        }
                    }
                }
            }
            return allowIndexingEvaluation;
        }, MetadataAccess.SERVICE);
        if (allowIndexing) {
            final Map<String, Object> fields = new HashMap<>();
            // Determine database and table names
            final Map<String, Object> properties = datasource.getProperties();
            fields.put("databaseName", properties.get("Target schema"));
            fields.put("tableName", properties.get("Target table"));
            // Generate list of column metadata
            final Map<String, Object> genericProperties = datasource.getGenericProperties();
            final Object columns = genericProperties.get("columns");
            if (columns != null && columns instanceof List) {
                final List<Map<String, Object>> hiveColumns = ((List<?>) columns).stream().map(Map.class::cast).map(map -> {
                    final Map<String, Object> column = new HashMap<>();
                    column.put("columnComment", map.get("description"));
                    column.put("columnName", map.get("name"));
                    @SuppressWarnings("unchecked") final List<Map<String, String>> tags = (List<Map<String, String>>) map.get("tags");
                    if (tags != null && !tags.isEmpty()) {
                        column.put("columnTags", tags.stream().map(tag -> tag.get("name")).collect(Collectors.toList()));
                    }
                    column.put("columnType", map.get("derivedDataType"));
                    return column;
                }).collect(Collectors.toList());
                fields.put("hiveColumns", hiveColumns);
            }
            // Index the Hive schema
            if (fields.get("databaseName") != null && fields.get("tableName") != null) {
                search.index(SearchIndex.DATASOURCES, datasource.getDatasourceType(), datasource.getId().toString(), fields);
                return true;
            }
        } else {
            // Drop schema from index if feed's indexing is disabled
            try {
                return checkAndDeleteSchema(((JcrDerivedDatasource) datasource).getId().getIdValue(), ((JcrDerivedDatasource) datasource).getPath());
            } catch (RepositoryException e) {
                log.warn("Unable to get id and/or path for datasource: {}", e.getMessage());
            }
        }
    }
    return false;
}