Examples with FeedDestination com.thinkbiganalytics.metadata.rest.model.feed.FeedDestination used on opensource projects

Example 1 with FeedDestination

use of com.thinkbiganalytics.metadata.rest.model.feed.FeedDestination in project kylo by Teradata.

the class MetadataClientProviderTest method testBeginOperation.

@Test
public void testBeginOperation() {
    Feed feed = this.provider.ensureFeed("category", "test9", "");
    Datasource ds = this.provider.ensureDirectoryDatasource("test9", "", Paths.get("aaa", "bbb"));
    feed = this.provider.ensureFeedDestination(feed.getId(), ds.getId());
    FeedDestination dest = feed.getDestination(ds.getId());
    DataOperation op = this.provider.beginOperation(dest, new DateTime());
    assertThat(op).isNotNull();
    assertThat(op.getState()).isEqualTo(State.IN_PROGRESS);
}

Example 2 with FeedDestination

use of com.thinkbiganalytics.metadata.rest.model.feed.FeedDestination in project kylo by Teradata.

the class FeedsController method ensureDependentDatasources.

private void ensureDependentDatasources(Feed feed, com.thinkbiganalytics.metadata.api.feed.Feed domainFeed) {
    for (FeedSource src : feed.getSources()) {
        Datasource.ID dsId = this.datasetProvider.resolve(src.getId());
        feedProvider.ensureFeedSource(domainFeed.getId(), dsId);
    }
    for (FeedDestination src : feed.getDestinations()) {
        Datasource.ID dsId = this.datasetProvider.resolve(src.getId());
        feedProvider.ensureFeedDestination(domainFeed.getId(), dsId);
    }
}

Example 3 with FeedDestination

use of com.thinkbiganalytics.metadata.rest.model.feed.FeedDestination in project kylo by Teradata.

the class Model method domainToFeed.

/**
 * Transforms the specified domain object to a REST object.
 *
 * @param domain     the domain object
 * @param addSources {@code true} to include sources in the REST object, or {@code false} otherwise
 * @return the REST object
 */
public Feed domainToFeed(@Nonnull final com.thinkbiganalytics.metadata.api.feed.Feed domain, final boolean addSources) {
    Feed feed = new Feed();
    feed.setId(domain.getId().toString());
    feed.setSystemName(domain.getName());
    feed.setDisplayName(domain.getDisplayName());
    feed.setDescription(domain.getDescription());
    feed.setState(Feed.State.valueOf(domain.getState().name()));
    feed.setCreatedTime(domain.getCreatedTime());
    feed.setCurrentInitStatus(DOMAIN_TO_INIT_STATUS.apply(domain.getCurrentInitStatus()));
    if (domain.getCategory() != null) {
        feed.setCategory(DOMAIN_TO_FEED_CATEGORY.apply(domain.getCategory()));
    }
    if (addSources) {
        @SuppressWarnings("unchecked") Collection<FeedSource> sources = Collections2.transform(domain.getSources(), this::domainToFeedSource);
        feed.setSources(new HashSet<>(sources));
        @SuppressWarnings("unchecked") Collection<FeedDestination> destinations = Collections2.transform(domain.getDestinations(), this::domainToFeedDestination);
        feed.setDestinations(new HashSet<>(destinations));
    }
    for (Entry<String, Object> entry : domain.getProperties().entrySet()) {
        if (entry.getValue() != null) {
            feed.getProperties().setProperty(entry.getKey(), entry.getValue().toString());
        }
    }
    return feed;
}

Example 4 with FeedDestination

use of com.thinkbiganalytics.metadata.rest.model.feed.FeedDestination in project kylo by Teradata.

the class AbstractTerminateFeed method onTrigger.

/* (non-Javadoc)
     * @see org.apache.nifi.processor.AbstractProcessor#onTrigger(org.apache.nifi.processor.ProcessContext, org.apache.nifi.processor.ProcessSession)
     */
@Override
public void onTrigger(ProcessContext context, ProcessSession session) throws ProcessException {
    try {
        FlowFile flowFile = session.get();
        if (flowFile != null) {
            MetadataProvider provider = getProviderService(context).getProvider();
            // TODO Remove when we do more intelligent handling when the feed and datasource info has been
            // removed from the metadata store.
            // FeedDestination destination = this.feedDestination.get();
            Datasource ds = ensureDatasourceMetadata(context);
            FeedDestination destination = ensureFeedDestination(context, flowFile, ds);
            // TODO Begin re-caching this when the correct error can be surfaced from the client
            // when the destination no longer exists.
            // if (destination == null) {
            // Datasource ds = this.destinationDatasource.get();
            // this.feedDestination.compareAndSet(null, destination);
            // }
            String timeStr = flowFile.getAttribute(OPERATON_START_PROP);
            DateTime opStart = timeStr != null ? Formatters.parseDateTime(timeStr) : new DateTime();
            DataOperation op = provider.beginOperation(destination, opStart);
            op = completeOperation(context, flowFile, ds, op, getState(context, op));
            updateFeedState(context, flowFile);
            flowFile = session.putAttribute(flowFile, OPERATON_STOP_PROP, Formatters.print(new DateTime()));
            session.remove(flowFile);
        // session.transfer(flowFile, SUCCESS);
        } else {
            context.yield();
        }
    } catch (Exception e) {
        context.yield();
        session.rollback();
        getLog().error("Unexpected error processing feed completion", e);
        throw new ProcessException(e);
    }
}

Example 5 with FeedDestination

use of com.thinkbiganalytics.metadata.rest.model.feed.FeedDestination in project kylo by Teradata.

the class MetadataClientProviderTest method testCompleteOperation.

@Test
public void testCompleteOperation() {
    Feed feed = this.provider.ensureFeed("category", "test10", "");
    DirectoryDatasource ds = this.provider.ensureDirectoryDatasource("test10", "", Paths.get("aaa", "bbb"));
    feed = this.provider.ensureFeedDestination(feed.getId(), ds.getId());
    FeedDestination dest = feed.getDestination(ds.getId());
    DataOperation op = this.provider.beginOperation(dest, new DateTime());
    Dataset set = this.provider.createDataset(ds, Paths.get("a.txt"), Paths.get("b.txt"));
    op = this.provider.completeOperation(op.getId(), "", set);
    assertThat(op).isNotNull();
    assertThat(op.getState()).isEqualTo(State.SUCCESS);
}