Examples with JcrDerivedDatasource com.thinkbiganalytics.metadata.modeshape.datasource.JcrDerivedDatasource used on opensource projects

Example 1 with JcrDerivedDatasource

use of com.thinkbiganalytics.metadata.modeshape.datasource.JcrDerivedDatasource in project kylo by Teradata.

the class JcrPropertyTest method testFeed.

/**
 * Creates a new Category Creates a new Feed Updates the Feed Get Feed and its versions Validates Feed is versioned along with its properties and can successfully return a version
 */
@Test
public void testFeed() {
    String categorySystemName = "my_category";
    Category category = metadata.commit(() -> {
        JcrCategory cat = (JcrCategory) categoryProvider.ensureCategory(categorySystemName);
        cat.setDescription("my category desc");
        cat.setTitle("my category");
        categoryProvider.update(cat);
        return cat;
    }, MetadataAccess.SERVICE);
    final JcrFeed.FeedId createdFeedId = metadata.commit(() -> {
        String sysName = "my_category";
        JcrCategory cat = (JcrCategory) categoryProvider.ensureCategory(sysName);
        cat.setDescription("my category desc");
        cat.setTitle("my category");
        categoryProvider.update(cat);
        JcrDerivedDatasource datasource1 = (JcrDerivedDatasource) datasourceProvider.ensureDerivedDatasource("HiveDatasource", "mysql.table1", "mysql.table1", "mysql table source 1", null);
        JcrDerivedDatasource datasource2 = (JcrDerivedDatasource) datasourceProvider.ensureDerivedDatasource("HiveDatasource", "mysql.table2", "mysql.table2", "mysql table source 2", null);
        JcrDerivedDatasource emptySource1 = (JcrDerivedDatasource) datasourceProvider.ensureDerivedDatasource("HDFSDatasource", "", "", "empty hdfs source", null);
        JcrDerivedDatasource emptySource2 = (JcrDerivedDatasource) datasourceProvider.ensureDerivedDatasource("HDFSDatasource", "", "", "empty hdfs source", null);
        Assert.assertEquals(emptySource1.getId(), emptySource2.getId());
        JcrDerivedDatasource emptySource3 = (JcrDerivedDatasource) datasourceProvider.ensureDerivedDatasource("HDFSDatasource", null, null, "empty hdfs source", null);
        JcrDerivedDatasource emptySource4 = (JcrDerivedDatasource) datasourceProvider.ensureDerivedDatasource("HDFSDatasource", null, null, "empty hdfs source", null);
        Assert.assertEquals(emptySource3.getId(), emptySource4.getId());
        JcrDerivedDatasource datasource3 = (JcrDerivedDatasource) datasourceProvider.ensureDerivedDatasource("HDFSDatasource", "/etl/customers/swert/012320342", "/etl/customers/swert/012320342", "mysql hdfs source", null);
        JcrDerivedDatasource datasource4 = (JcrDerivedDatasource) datasourceProvider.ensureDerivedDatasource("HDFSDatasource", "/etl/customers/swert/012320342", "/etl/customers/swert/012320342", "mysql hdfs source", null);
        Assert.assertEquals(datasource3.getId(), datasource4.getId());
        String feedSystemName = "my_feed_" + UUID.randomUUID();
        // JcrFeed feed = (JcrFeed) feedProvider.ensureFeed(categorySystemName, feedSystemName, "my feed desc", datasource1.getId(), null);
        JcrFeed feed = (JcrFeed) feedProvider.ensureFeed(sysName, feedSystemName, "my feed desc");
        feedProvider.ensureFeedSource(feed.getId(), datasource1.getId());
        feedProvider.ensureFeedSource(feed.getId(), datasource2.getId());
        feed.setTitle("my feed");
        feed.addTag("my tag");
        feed.addTag("my second tag");
        feed.addTag("feedTag");
        Map<String, Object> otherProperties = new HashMap<String, Object>();
        otherProperties.put("prop1", "my prop1");
        otherProperties.put("prop2", "my prop2");
        feed.setProperties(otherProperties);
        return feed.getId();
    }, MetadataAccess.SERVICE);
    // read and find feed verisons and ensure props
    JcrFeed.FeedId readFeedId = metadata.read(() -> {
        Session s = null;
        JcrFeed f = (JcrFeed) ((JcrFeedProvider) feedProvider).findById(createdFeedId);
        // TODO: Feed vesioning disabled for Kylo v0.5.0
        // int versions = printVersions(f);
        // Assert.assertTrue(versions > 1, "Expecting more than 1 version: jcr:rootVersion, 1.0");
        @SuppressWarnings("unchecked") List<? extends FeedSource> sources = f.getSources();
        Assert.assertTrue(sources.size() > 0);
        if (sources != null) {
            for (FeedSource source : sources) {
                Map<String, Object> dataSourceProperties = ((JcrDatasource) source.getDatasource()).getAllProperties();
                String type = (String) dataSourceProperties.get(JcrDerivedDatasource.TYPE_NAME);
                Assert.assertEquals(type, "HiveDatasource");
            }
        }
        List<JcrObject> taggedObjects = tagProvider.findByTag("my tag");
        // assert we got 1 feed back
        Assert.assertTrue(taggedObjects.size() >= 1);
        return f.getId();
    }, MetadataAccess.SERVICE);
    // update the feed again
    JcrFeed.FeedId updatedFeed = metadata.commit(() -> {
        JcrFeed f = (JcrFeed) ((JcrFeedProvider) feedProvider).findById(createdFeedId);
        f.setDescription("My Feed Updated Description");
        Map<String, Object> otherProperties = new HashMap<String, Object>();
        otherProperties.put("prop1", "my updated prop1");
        f.setProperties(otherProperties);
        ((JcrFeedProvider) feedProvider).update(f);
        return f.getId();
    }, MetadataAccess.SERVICE);
    // read it again and find the versions
    readFeedId = metadata.read(() -> {
        JcrFeed f = (JcrFeed) ((JcrFeedProvider) feedProvider).findById(updatedFeed);
        // Assert.assertEquals(v1Id, baseId);
        return f.getId();
    }, MetadataAccess.SERVICE);
}

Example 2 with JcrDerivedDatasource

use of com.thinkbiganalytics.metadata.modeshape.datasource.JcrDerivedDatasource in project kylo by Teradata.

the class JcrIndexService method indexDerivedDatasource.

/**
 * Indexes derived datasource objects.
 *
 * @param datasource the derived datasource to index
 * @return {@code true} if the index was updated, or {@code false} otherwise
 */
private boolean indexDerivedDatasource(@Nonnull final DerivedDatasource datasource) {
    if (HIVE_DATASOURCE.equals(datasource.getDatasourceType())) {
        boolean allowIndexing = metadataAccess.read(() -> {
            boolean allowIndexingEvaluation = true;
            Session session = JcrMetadataAccess.getActiveSession();
            if (session != null) {
                Value[] feedDestinationsArray = ((JcrDerivedDatasource) datasource).getNode().getProperty("tba:feedDestinations").getValues();
                for (Value feedDestination : feedDestinationsArray) {
                    Node feedDestinationNode = session.getNodeByIdentifier(feedDestination.getString());
                    if (feedDestinationNode != null) {
                        Node feedDetailsNode = feedDestinationNode.getParent();
                        if (feedDetailsNode != null) {
                            Node feedSummaryNode = feedDetailsNode.getParent();
                            if (feedSummaryNode != null) {
                                String indexingOption = feedSummaryNode.getProperty("tba:allowIndexing").getString();
                                if ((indexingOption != null) && (indexingOption.equals("N"))) {
                                    allowIndexingEvaluation = false;
                                }
                            }
                        }
                    }
                }
            }
            return allowIndexingEvaluation;
        }, MetadataAccess.SERVICE);
        if (allowIndexing) {
            final Map<String, Object> fields = new HashMap<>();
            // Determine database and table names
            final Map<String, Object> properties = datasource.getProperties();
            fields.put("databaseName", properties.get("Target schema"));
            fields.put("tableName", properties.get("Target table"));
            // Generate list of column metadata
            final Map<String, Object> genericProperties = datasource.getGenericProperties();
            final Object columns = genericProperties.get("columns");
            if (columns != null && columns instanceof List) {
                final List<Map<String, Object>> hiveColumns = ((List<?>) columns).stream().map(Map.class::cast).map(map -> {
                    final Map<String, Object> column = new HashMap<>();
                    column.put("columnComment", map.get("description"));
                    column.put("columnName", map.get("name"));
                    @SuppressWarnings("unchecked") final List<Map<String, String>> tags = (List<Map<String, String>>) map.get("tags");
                    if (tags != null && !tags.isEmpty()) {
                        column.put("columnTags", tags.stream().map(tag -> tag.get("name")).collect(Collectors.toList()));
                    }
                    column.put("columnType", map.get("derivedDataType"));
                    return column;
                }).collect(Collectors.toList());
                fields.put("hiveColumns", hiveColumns);
            }
            // Index the Hive schema
            if (fields.get("databaseName") != null && fields.get("tableName") != null) {
                search.index(SearchIndex.DATASOURCES, datasource.getDatasourceType(), datasource.getId().toString(), fields);
                return true;
            }
        } else {
            // Drop schema from index if feed's indexing is disabled
            try {
                return checkAndDeleteSchema(((JcrDerivedDatasource) datasource).getId().getIdValue(), ((JcrDerivedDatasource) datasource).getPath());
            } catch (RepositoryException e) {
                log.warn("Unable to get id and/or path for datasource: {}", e.getMessage());
            }
        }
    }
    return false;
}