Examples with JcrDatasource com.thinkbiganalytics.metadata.modeshape.datasource.JcrDatasource used on opensource projects

Example 1 with JcrDatasource

use of com.thinkbiganalytics.metadata.modeshape.datasource.JcrDatasource in project kylo by Teradata.

the class FeedDetails method removeFeedDestinations.

protected void removeFeedDestinations() {
    List<? extends FeedDestination> destinations = getDestinations();
    if (destinations != null && !destinations.isEmpty()) {
        destinations.stream().forEach(dest -> {
            try {
                // Remove the connection nodes
                Node destNode = ((JcrFeedDestination) dest).getNode();
                JcrDatasource datasource = (JcrDatasource) dest.getDatasource();
                datasource.removeDestinationNode(destNode);
                destNode.remove();
                // Remove the datasource if there are no referencing feeds
                if (datasource.getFeedDestinations().isEmpty() && datasource.getFeedSources().isEmpty()) {
                    datasource.remove();
                }
            } catch (RepositoryException e) {
                e.printStackTrace();
            }
        });
    }
}

Example 2 with JcrDatasource

use of com.thinkbiganalytics.metadata.modeshape.datasource.JcrDatasource in project kylo by Teradata.

the class MigrateLegacyDatasourcesUpgradeAction method upgradeTo.

@Override
public void upgradeTo(final KyloVersion targetVersion) {
    log.info("Setting up catalog access control: {}", targetVersion);
    try {
        NodeTypeManager typeMgr = JcrMetadataAccess.getActiveSession().getWorkspace().getNodeTypeManager();
        NodeType dataSourceType = typeMgr.getNodeType("tba:DataSource");
        Connector conn = pluginManager.getPlugin("jdbc").flatMap(plugin -> connectorProvider.findByPlugin(plugin.getId())).orElseThrow(() -> new IllegalStateException("No JDBC connector found"));
        legacyProvider.getDatasources().stream().map(JcrDatasource.class::cast).map(JcrDatasource::getNode).filter(node -> JcrUtil.isNodeType(node, "tba:userDatasource")).filter(node -> JcrUtil.isNodeType(JcrUtil.getNode(node, "tba:details"), "tba:jdbcDatasourceDetails")).forEach(jdbcNode -> {
            try {
                // Move the legacy datasource node to a temporary parent that doesn't have any constraints (the catalog folder).
                Node tmpParent = JcrUtil.getNode(JcrUtil.getRootNode(jdbcNode), "metadata/catalog");
                String dsSystemName = dataSourceProvider.generateSystemName(JcrUtil.getName(jdbcNode));
                Path catDsPath = MetadataPaths.dataSourcePath(conn.getSystemName(), dsSystemName);
                Node catDsNode = JcrUtil.moveNode(jdbcNode, JcrUtil.path(tmpParent.getPath()).resolve(dsSystemName));
                // Wrap the node as a legacy UserDataSource and collect its properties.
                JcrUserDatasource legacyDs = JcrUtil.getJcrObject(catDsNode, JcrUserDatasource.class);
                Set<? extends JcrFeed> referencingFeeds = legacyDs.getFeedSources().stream().map(FeedSource::getFeed).map(JcrFeed.class::cast).collect(Collectors.toSet());
                AtomicReference<String> controllerServiceId = new AtomicReference<>();
                AtomicReference<String> password = new AtomicReference<>();
                legacyDs.getDetails().filter(JcrJdbcDatasourceDetails.class::isInstance).map(JcrJdbcDatasourceDetails.class::cast).ifPresent(details -> {
                    controllerServiceId.set(details.getControllerServiceId().orElse(null));
                    password.set("{cipher}" + details.getPassword());
                });
                if (this.accessController.isEntityAccessControlled()) {
                    legacyDs.disableAccessControl(legacyDs.getOwner());
                }
                // Convert the legacy type into the catalog type.
                JcrDataSource catDs = convertToDataSource(catDsNode, catDsPath, dataSourceType, controllerServiceId.get(), password.get());
                linkDataSets(catDs, referencingFeeds);
                if (this.accessController.isEntityAccessControlled()) {
                    List<SecurityRole> roles = roleProvider.getEntityRoles(SecurityRole.DATASOURCE);
                    actionsProvider.getAvailableActions(AllowedActions.DATASOURCE).ifPresent(actions -> catDs.enableAccessControl((JcrAllowedActions) actions, legacyDs.getOwner(), roles));
                }
            } catch (RepositoryException e) {
                throw new UpgradeException("Failed to migrate legacy datasources", e);
            }
        });
    } catch (IllegalStateException | RepositoryException e) {
        throw new UpgradeException("Failed to migrate legacy datasources", e);
    }
}

Example 3 with JcrDatasource

use of com.thinkbiganalytics.metadata.modeshape.datasource.JcrDatasource in project kylo by Teradata.

the class JcrPropertyTest method testFeed.

/**
 * Creates a new Category Creates a new Feed Updates the Feed Get Feed and its versions Validates Feed is versioned along with its properties and can successfully return a version
 */
@Test
public void testFeed() {
    String categorySystemName = "my_category";
    Category category = metadata.commit(() -> {
        JcrCategory cat = (JcrCategory) categoryProvider.ensureCategory(categorySystemName);
        cat.setDescription("my category desc");
        cat.setTitle("my category");
        categoryProvider.update(cat);
        return cat;
    }, MetadataAccess.SERVICE);
    final JcrFeed.FeedId createdFeedId = metadata.commit(() -> {
        String sysName = "my_category";
        JcrCategory cat = (JcrCategory) categoryProvider.ensureCategory(sysName);
        cat.setDescription("my category desc");
        cat.setTitle("my category");
        categoryProvider.update(cat);
        JcrDerivedDatasource datasource1 = (JcrDerivedDatasource) datasourceProvider.ensureDerivedDatasource("HiveDatasource", "mysql.table1", "mysql.table1", "mysql table source 1", null);
        JcrDerivedDatasource datasource2 = (JcrDerivedDatasource) datasourceProvider.ensureDerivedDatasource("HiveDatasource", "mysql.table2", "mysql.table2", "mysql table source 2", null);
        JcrDerivedDatasource emptySource1 = (JcrDerivedDatasource) datasourceProvider.ensureDerivedDatasource("HDFSDatasource", "", "", "empty hdfs source", null);
        JcrDerivedDatasource emptySource2 = (JcrDerivedDatasource) datasourceProvider.ensureDerivedDatasource("HDFSDatasource", "", "", "empty hdfs source", null);
        Assert.assertEquals(emptySource1.getId(), emptySource2.getId());
        JcrDerivedDatasource emptySource3 = (JcrDerivedDatasource) datasourceProvider.ensureDerivedDatasource("HDFSDatasource", null, null, "empty hdfs source", null);
        JcrDerivedDatasource emptySource4 = (JcrDerivedDatasource) datasourceProvider.ensureDerivedDatasource("HDFSDatasource", null, null, "empty hdfs source", null);
        Assert.assertEquals(emptySource3.getId(), emptySource4.getId());
        JcrDerivedDatasource datasource3 = (JcrDerivedDatasource) datasourceProvider.ensureDerivedDatasource("HDFSDatasource", "/etl/customers/swert/012320342", "/etl/customers/swert/012320342", "mysql hdfs source", null);
        JcrDerivedDatasource datasource4 = (JcrDerivedDatasource) datasourceProvider.ensureDerivedDatasource("HDFSDatasource", "/etl/customers/swert/012320342", "/etl/customers/swert/012320342", "mysql hdfs source", null);
        Assert.assertEquals(datasource3.getId(), datasource4.getId());
        String feedSystemName = "my_feed_" + UUID.randomUUID();
        // JcrFeed feed = (JcrFeed) feedProvider.ensureFeed(categorySystemName, feedSystemName, "my feed desc", datasource1.getId(), null);
        JcrFeed feed = (JcrFeed) feedProvider.ensureFeed(sysName, feedSystemName, "my feed desc");
        feedProvider.ensureFeedSource(feed.getId(), datasource1.getId());
        feedProvider.ensureFeedSource(feed.getId(), datasource2.getId());
        feed.setTitle("my feed");
        feed.addTag("my tag");
        feed.addTag("my second tag");
        feed.addTag("feedTag");
        Map<String, Object> otherProperties = new HashMap<String, Object>();
        otherProperties.put("prop1", "my prop1");
        otherProperties.put("prop2", "my prop2");
        feed.setProperties(otherProperties);
        return feed.getId();
    }, MetadataAccess.SERVICE);
    // read and find feed verisons and ensure props
    JcrFeed.FeedId readFeedId = metadata.read(() -> {
        Session s = null;
        JcrFeed f = (JcrFeed) ((JcrFeedProvider) feedProvider).findById(createdFeedId);
        // TODO: Feed vesioning disabled for Kylo v0.5.0
        // int versions = printVersions(f);
        // Assert.assertTrue(versions > 1, "Expecting more than 1 version: jcr:rootVersion, 1.0");
        @SuppressWarnings("unchecked") List<? extends FeedSource> sources = f.getSources();
        Assert.assertTrue(sources.size() > 0);
        if (sources != null) {
            for (FeedSource source : sources) {
                Map<String, Object> dataSourceProperties = ((JcrDatasource) source.getDatasource().get()).getAllProperties();
                String type = (String) dataSourceProperties.get(JcrDerivedDatasource.TYPE_NAME);
                Assert.assertEquals(type, "HiveDatasource");
            }
        }
        List<JcrObject> taggedObjects = tagProvider.findByTag("my tag");
        // assert we got 1 feed back
        Assert.assertTrue(taggedObjects.size() >= 1);
        return f.getId();
    }, MetadataAccess.SERVICE);
    // update the feed again
    JcrFeed.FeedId updatedFeed = metadata.commit(() -> {
        JcrFeed f = (JcrFeed) ((JcrFeedProvider) feedProvider).findById(createdFeedId);
        f.setDescription("My Feed Updated Description");
        Map<String, Object> otherProperties = new HashMap<String, Object>();
        otherProperties.put("prop1", "my updated prop1");
        f.setProperties(otherProperties);
        ((JcrFeedProvider) feedProvider).update(f);
        return f.getId();
    }, MetadataAccess.SERVICE);
    // read it again and find the versions
    readFeedId = metadata.read(() -> {
        JcrFeed f = (JcrFeed) ((JcrFeedProvider) feedProvider).findById(updatedFeed);
        // Assert.assertEquals(v1Id, baseId);
        return f.getId();
    }, MetadataAccess.SERVICE);
}

Example 4 with JcrDatasource

use of com.thinkbiganalytics.metadata.modeshape.datasource.JcrDatasource in project kylo by Teradata.

the class JcrFeedProvider method ensureFeedDestination.

@Override
public FeedDestination ensureFeedDestination(Feed.ID feedId, com.thinkbiganalytics.metadata.api.datasource.Datasource.ID dsId) {
    JcrFeed feed = (JcrFeed) findById(feedId);
    FeedDestination source = feed.getDestination(dsId);
    if (source == null) {
        JcrDatasource datasource = (JcrDatasource) datasourceProvider.getDatasource(dsId);
        if (datasource != null) {
            JcrFeedDestination jcrDest = feed.ensureFeedDestination(datasource);
            // save();
            return jcrDest;
        } else {
            throw new DatasourceNotFoundException(dsId);
        }
    } else {
        return source;
    }
}

Example 5 with JcrDatasource

use of com.thinkbiganalytics.metadata.modeshape.datasource.JcrDatasource in project kylo by Teradata.

the class FeedDetails method removeFeedSources.

protected void removeFeedSources() {
    List<? extends FeedSource> sources = getSources();
    if (sources != null && !sources.isEmpty()) {
        // checkout the feed
        sources.stream().forEach(source -> {
            try {
                Node sourceNode = ((JcrFeedSource) source).getNode();
                ((JcrDatasource) ((JcrFeedSource) source).getDatasource()).removeSourceNode(sourceNode);
                sourceNode.remove();
            } catch (RepositoryException e) {
                e.printStackTrace();
            }
        });
    }
}