Examples with MetadataProvider com.thinkbiganalytics.nifi.core.api.metadata.MetadataProvider used on opensource projects

Example 11 with MetadataProvider

use of com.thinkbiganalytics.nifi.core.api.metadata.MetadataProvider in project kylo by Teradata.

the class AbstractTerminateFeed method updateFeedState.

private void updateFeedState(ProcessContext context, FlowFile flowFile) {
    MetadataProvider provider = getProviderService(context).getProvider();
    String feedId = flowFile.getAttribute(FEED_ID_PROP);
    if (feedId != null) {
        Properties props = deriveFeedProperties(flowFile);
        provider.updateFeedProperties(feedId, props);
    }
}

Example 12 with MetadataProvider

use of com.thinkbiganalytics.nifi.core.api.metadata.MetadataProvider in project kylo by Teradata.

the class AbstractTerminateFeed method onTrigger.

/* (non-Javadoc)
     * @see org.apache.nifi.processor.AbstractProcessor#onTrigger(org.apache.nifi.processor.ProcessContext, org.apache.nifi.processor.ProcessSession)
     */
@Override
public void onTrigger(ProcessContext context, ProcessSession session) throws ProcessException {
    try {
        FlowFile flowFile = session.get();
        if (flowFile != null) {
            MetadataProvider provider = getProviderService(context).getProvider();
            // TODO Remove when we do more intelligent handling when the feed and datasource info has been
            // removed from the metadata store.
            // FeedDestination destination = this.feedDestination.get();
            Datasource ds = ensureDatasourceMetadata(context);
            FeedDestination destination = ensureFeedDestination(context, flowFile, ds);
            // TODO Begin re-caching this when the correct error can be surfaced from the client
            // when the destination no longer exists.
            // if (destination == null) {
            // Datasource ds = this.destinationDatasource.get();
            // this.feedDestination.compareAndSet(null, destination);
            // }
            String timeStr = flowFile.getAttribute(OPERATON_START_PROP);
            DateTime opStart = timeStr != null ? Formatters.parseDateTime(timeStr) : new DateTime();
            DataOperation op = provider.beginOperation(destination, opStart);
            op = completeOperation(context, flowFile, ds, op, getState(context, op));
            updateFeedState(context, flowFile);
            flowFile = session.putAttribute(flowFile, OPERATON_STOP_PROP, Formatters.print(new DateTime()));
            session.remove(flowFile);
        // session.transfer(flowFile, SUCCESS);
        } else {
            context.yield();
        }
    } catch (Exception e) {
        context.yield();
        session.rollback();
        getLog().error("Unexpected error processing feed completion", e);
        throw new ProcessException(e);
    }
}

Example 13 with MetadataProvider

use of com.thinkbiganalytics.nifi.core.api.metadata.MetadataProvider in project kylo by Teradata.

the class TerminateDirectoryFeed method createDestinationDatasource.

@Override
protected Datasource createDestinationDatasource(ProcessContext context, String datasetName, String descr) {
    MetadataProvider provider = getProviderService(context).getProvider();
    String path = context.getProperty(DirectoryProperties.DIRECTORY_PATH).getValue();
    return provider.ensureDirectoryDatasource(datasetName, "", Paths.get(path));
}

Example 14 with MetadataProvider

use of com.thinkbiganalytics.nifi.core.api.metadata.MetadataProvider in project kylo by Teradata.

the class ExecuteSparkJob method fetchCatalogDataSource.

/**
 * Fetches a catalog datasource and populates the extraJars if found
 */
private com.thinkbiganalytics.kylo.catalog.rest.model.DataSource fetchCatalogDataSource(String id, ProcessSession session, FlowFile flowFile, MetadataProviderService metadataService, List<String> extraJarPaths) {
    final MetadataProvider provider = metadataService.getProvider();
    final Optional<com.thinkbiganalytics.kylo.catalog.rest.model.DataSource> optionalDataSource;
    try {
        optionalDataSource = provider.getCatalogDataSource(id);
    } catch (final Exception e) {
        getLog().error("Unable to access catalog data source: {}: {}", new Object[] { id, e }, e);
        throw e;
    }
    if (optionalDataSource.isPresent()) {
        com.thinkbiganalytics.kylo.catalog.rest.model.DataSource dataSource = optionalDataSource.get();
        if (dataSource.getTemplate() != null && dataSource.getTemplate().getJars() != null) {
            extraJarPaths.addAll(dataSource.getTemplate().getJars());
        }
        if (dataSource.getConnector() != null && dataSource.getConnector().getTemplate() != null && dataSource.getConnector().getTemplate().getJars() != null) {
            extraJarPaths.addAll(dataSource.getConnector().getTemplate().getJars());
        }
    }
    return optionalDataSource != null && optionalDataSource.isPresent() ? optionalDataSource.get() : null;
}

Example 15 with MetadataProvider

use of com.thinkbiganalytics.nifi.core.api.metadata.MetadataProvider in project kylo by Teradata.

the class ExecuteSparkJob method fetchDatasource.

/**
 * fetches a legacy datasource and populates the extraJars if found
 */
private Datasource fetchDatasource(String id, ProcessSession session, FlowFile flowFile, MetadataProviderService metadataService, List<String> extraJarPaths) {
    final MetadataProvider provider = metadataService.getProvider();
    final Optional<Datasource> datasource;
    try {
        datasource = provider.getDatasource(id);
    } catch (final Exception e) {
        getLog().error("Unable to access data source: {}: {}", new Object[] { id, e }, e);
        throw e;
    }
    if (datasource.isPresent()) {
        if (datasource.get() instanceof JdbcDatasource && StringUtils.isNotBlank(((JdbcDatasource) datasource.get()).getDatabaseDriverLocation())) {
            final String[] databaseDriverLocations = ((JdbcDatasource) datasource.get()).getDatabaseDriverLocation().split(",");
            extraJarPaths.addAll(Arrays.asList(databaseDriverLocations));
        }
    }
    return datasource != null && datasource.isPresent() ? datasource.get() : null;
}