Examples with DataOperation com.thinkbiganalytics.metadata.rest.model.op.DataOperation used on opensource projects

Example 6 with DataOperation

use of com.thinkbiganalytics.metadata.rest.model.op.DataOperation in project kylo by Teradata.

the class MetadataClientProviderTest method testCompleteOperation.

@Test
public void testCompleteOperation() {
    Feed feed = this.provider.ensureFeed("category", "test10", "");
    DirectoryDatasource ds = this.provider.ensureDirectoryDatasource("test10", "", Paths.get("aaa", "bbb"));
    feed = this.provider.ensureFeedDestination(feed.getId(), ds.getId());
    FeedDestination dest = feed.getDestination(ds.getId());
    DataOperation op = this.provider.beginOperation(dest, new DateTime());
    Dataset set = this.provider.createDataset(ds, Paths.get("a.txt"), Paths.get("b.txt"));
    op = this.provider.completeOperation(op.getId(), "", set);
    assertThat(op).isNotNull();
    assertThat(op.getState()).isEqualTo(State.SUCCESS);
}

Example 7 with DataOperation

use of com.thinkbiganalytics.metadata.rest.model.op.DataOperation in project kylo by Teradata.

the class AbstractTerminateFeed method onTrigger.

/* (non-Javadoc)
     * @see org.apache.nifi.processor.AbstractProcessor#onTrigger(org.apache.nifi.processor.ProcessContext, org.apache.nifi.processor.ProcessSession)
     */
@Override
public void onTrigger(ProcessContext context, ProcessSession session) throws ProcessException {
    try {
        FlowFile flowFile = session.get();
        if (flowFile != null) {
            MetadataProvider provider = getProviderService(context).getProvider();
            // TODO Remove when we do more intelligent handling when the feed and datasource info has been
            // removed from the metadata store.
            // FeedDestination destination = this.feedDestination.get();
            Datasource ds = ensureDatasourceMetadata(context);
            FeedDestination destination = ensureFeedDestination(context, flowFile, ds);
            // TODO Begin re-caching this when the correct error can be surfaced from the client
            // when the destination no longer exists.
            // if (destination == null) {
            // Datasource ds = this.destinationDatasource.get();
            // this.feedDestination.compareAndSet(null, destination);
            // }
            String timeStr = flowFile.getAttribute(OPERATON_START_PROP);
            DateTime opStart = timeStr != null ? Formatters.parseDateTime(timeStr) : new DateTime();
            DataOperation op = provider.beginOperation(destination, opStart);
            op = completeOperation(context, flowFile, ds, op, getState(context, op));
            updateFeedState(context, flowFile);
            flowFile = session.putAttribute(flowFile, OPERATON_STOP_PROP, Formatters.print(new DateTime()));
            session.remove(flowFile);
        // session.transfer(flowFile, SUCCESS);
        } else {
            context.yield();
        }
    } catch (Exception e) {
        context.yield();
        session.rollback();
        getLog().error("Unexpected error processing feed completion", e);
        throw new ProcessException(e);
    }
}

Example 8 with DataOperation

use of com.thinkbiganalytics.metadata.rest.model.op.DataOperation in project kylo by Teradata.

the class MetadataClientTest method testCheckPrecondition.

// @Test
public void testCheckPrecondition() throws ParseException {
    Feed feedA = buildFeed("category", "feedA").post();
    Feed feedB = buildFeed("category", "feedB", "category", "feedA").post();
    HiveTableDatasource dsA = buildHiveTableDatasource("test-table").post();
    feedA = client.addDestination(feedA.getId(), dsA.getId());
    String destIdA = feedA.getDestinations().iterator().next().getId();
    DataOperation op = client.beginOperation(destIdA, "");
    op.setState(State.SUCCESS);
    op.setDataset(new Dataset(new DateTime(), dsA, ChangeType.UPDATE, ContentType.PARTITIONS));
    op = client.updateDataOperation(op);
    ServiceLevelAssessment assmt = client.assessPrecondition(feedB.getId());
    assertThat(assmt).isNotNull();
}