Examples with MetadataProvider com.thinkbiganalytics.nifi.core.api.metadata.MetadataProvider used on opensource projects

Example 1 with MetadataProvider

use of com.thinkbiganalytics.nifi.core.api.metadata.MetadataProvider in project kylo by Teradata.

the class ExecuteSparkJob method getDatasources.

private Map<String, String> getDatasources(ProcessSession session, FlowFile flowFile, String PROVENANCE_JOB_STATUS_KEY, String datasourceIds, MetadataProviderService metadataService, List<String> extraJarPaths) throws JsonProcessingException {
    final Map<String, String> env = new HashMap<>();
    if (StringUtils.isNotBlank(datasourceIds)) {
        final StringBuilder datasources = new StringBuilder(10240);
        final ObjectMapper objectMapper = new ObjectMapper();
        final MetadataProvider provider = metadataService.getProvider();
        for (final String id : datasourceIds.split(",")) {
            datasources.append((datasources.length() == 0) ? '[' : ',');
            final Optional<Datasource> datasource = provider.getDatasource(id);
            if (datasource.isPresent()) {
                if (datasource.get() instanceof JdbcDatasource && StringUtils.isNotBlank(((JdbcDatasource) datasource.get()).getDatabaseDriverLocation())) {
                    final String[] databaseDriverLocations = ((JdbcDatasource) datasource.get()).getDatabaseDriverLocation().split(",");
                    extraJarPaths.addAll(Arrays.asList(databaseDriverLocations));
                }
                datasources.append(objectMapper.writeValueAsString(datasource.get()));
            } else {
                getLog().error("Required datasource {} is missing for Spark job: {}", new Object[] { id, flowFile });
                flowFile = session.putAttribute(flowFile, PROVENANCE_JOB_STATUS_KEY, "Invalid data source: " + id);
                session.transfer(flowFile, REL_FAILURE);
                return null;
            }
        }
        datasources.append(']');
        env.put("DATASOURCES", datasources.toString());
    }
    return env;
}

Example 2 with MetadataProvider

use of com.thinkbiganalytics.nifi.core.api.metadata.MetadataProvider in project kylo by Teradata.

the class BeginFeed method setupSource.

protected void setupSource(ProcessContext context, Feed feed, String datasourceName) {
    MetadataProvider provider = getProviderService(context).getProvider();
    Datasource datasource = getSourceDatasource(context, datasourceName);
    if (datasource != null) {
        getLog().debug("ensuring feed source - feed: {} datasource: {}", new Object[] { feed.getId(), datasource.getId() });
        provider.ensureFeedSource(feed.getId(), datasource.getId());
    // this.sourceDatasources.add(datasource);
    } else {
        throw new ProcessException("Source datasource does not exist: " + datasourceName);
    }
}

Example 3 with MetadataProvider

use of com.thinkbiganalytics.nifi.core.api.metadata.MetadataProvider in project kylo by Teradata.

the class BeginFeed method ensureFeedMetadata.

@OnScheduled
public Feed ensureFeedMetadata(ProcessContext context) {
    MetadataProvider provider = getProviderService(context).getProvider();
    String feedName = context.getProperty(FEED_NAME).getValue();
    String categoryName = context.getProperty(CATEGORY_NAME).getValue();
    Feed feed = provider.ensureFeed(categoryName, feedName, "");
    // this.feedId.set(feed.getId());
    String datasourcesName = context.getProperty(SRC_DATASOURCES_NAME).getValue();
    if (!StringUtils.isEmpty(datasourcesName)) {
        String[] dsNameArr = datasourcesName.split("\\s*,\\s*");
        for (String dsName : dsNameArr) {
            setupSource(context, feed, dsName.trim());
        }
        ensurePreconditon(context, feed, dsNameArr);
        ensurePreconditonListener(context, feed, dsNameArr);
    }
    return feed;
}

Example 4 with MetadataProvider

use of com.thinkbiganalytics.nifi.core.api.metadata.MetadataProvider in project kylo by Teradata.

the class BeginFeed method ensurePreconditon.

private void ensurePreconditon(ProcessContext context, Feed feed, String[] dsNames) {
    MetadataProvider provider = getProviderService(context).getProvider();
    // If no precondition exits yet install one that depends on the datasources.
    if (feed.getPrecondition() == null) {
        getLog().debug("Setting default feed preconditions for: " + dsNames);
        Metric[] metrics = new Metric[dsNames.length];
        for (int idx = 0; idx < metrics.length; idx++) {
            DatasourceUpdatedSinceFeedExecuted metric = new DatasourceUpdatedSinceFeedExecuted(dsNames[idx], feed.getSystemName());
            metrics[idx] = metric;
        }
    // provider.ensurePrecondition(feed.getId(), metrics);
    }
}

Example 5 with MetadataProvider

use of com.thinkbiganalytics.nifi.core.api.metadata.MetadataProvider in project kylo by Teradata.

the class BeginFeed method ensurePreconditonListener.

private void ensurePreconditonListener(ProcessContext context, Feed feed, String[] dsNames) {
    if (this.preconditionListener == null) {
        MetadataProvider provider = getProviderService(context).getProvider();
        FeedPreconditionEventService precondService = getPreconditionService(context);
        PreconditionListener listener = new PreconditionListener() {

            @Override
            public void triggered(FeedPreconditionTriggerEvent event) {
                getLog().debug("Precondition event triggered: ", new Object[] { event });
                BeginFeed.this.pendingChanges.add(event);
            }
        };
        for (String dsName : dsNames) {
            getLog().debug("Adding precondition listener for datasoure name: " + dsName);
            precondService.addListener("", dsName, listener);
        }
        this.preconditionListener = listener;
    }
}