Examples with Dataset com.thinkbiganalytics.metadata.rest.model.op.Dataset used on opensource projects

Example 1 with Dataset

use of com.thinkbiganalytics.metadata.rest.model.op.Dataset in project kylo by Teradata.

the class TerminateHiveTableFeed method completeOperation.

@Override
protected DataOperation completeOperation(ProcessContext context, FlowFile flowFile, Datasource dataset, DataOperation op, DataOperation.State state) {
    MetadataProvider provider = getProviderService(context).getProvider();
    if (state == State.SUCCESS) {
        HiveTableDatasource hds = (HiveTableDatasource) dataset;
        Dataset changeSet = provider.createDataset(hds, null);
        return provider.completeOperation(op.getId(), "", changeSet);
    } else {
        return provider.completeOperation(op.getId(), "", state);
    }
}

Example 2 with Dataset

use of com.thinkbiganalytics.metadata.rest.model.op.Dataset in project kylo by Teradata.

the class MetadataClientTest method testCompleteOperation.

// @Test
public void testCompleteOperation() throws ParseException {
    Feed feedA = buildFeed("category", "feedA").post();
    HiveTableDatasource dsA = buildHiveTableDatasource("test-table").post();
    feedA = client.addDestination(feedA.getId(), dsA.getId());
    Feed feedB = buildFeed("category", "feedB", "category", "feedA").post();
    feedB = client.addSource(feedB.getId(), dsA.getId());
    String destA = feedA.getDestinations().iterator().next().getId();
    DataOperation op = client.beginOperation(destA, "");
    op.setState(State.SUCCESS);
    HiveTablePartitions changeSet = new HiveTablePartitions();
    changeSet.setPartitions(Arrays.asList(new HiveTablePartition("month", null, "Jan", "Feb"), new HiveTablePartition("year", null, "2015", "2016")));
    Dataset dataset = new Dataset(new DateTime(), dsA, ChangeType.UPDATE, ContentType.PARTITIONS, changeSet);
    op.setDataset(dataset);
    op = client.updateDataOperation(op);
    assertThat(op).isNotNull();
}

Example 3 with Dataset

use of com.thinkbiganalytics.metadata.rest.model.op.Dataset in project kylo by Teradata.

the class TerminateDirectoryFeed method completeOperation.

@Override
protected DataOperation completeOperation(ProcessContext context, FlowFile flowFile, Datasource datasource, DataOperation op, DataOperation.State state) {
    MetadataProvider provider = getProviderService(context).getProvider();
    DirectoryDatasource dds = (DirectoryDatasource) datasource;
    if (state == State.SUCCESS) {
        ArrayList<Path> paths = new ArrayList<>();
        // TODO Extract file paths from flow file
        Dataset dataset = provider.createDataset(dds, paths);
        return provider.completeOperation(op.getId(), "", dataset);
    } else {
        return provider.completeOperation(op.getId(), "", state);
    }
}

Example 4 with Dataset

use of com.thinkbiganalytics.metadata.rest.model.op.Dataset in project kylo by Teradata.

the class MetadataClientProviderTest method testCompleteOperation.

@Test
public void testCompleteOperation() {
    Feed feed = this.provider.ensureFeed("category", "test10", "");
    DirectoryDatasource ds = this.provider.ensureDirectoryDatasource("test10", "", Paths.get("aaa", "bbb"));
    feed = this.provider.ensureFeedDestination(feed.getId(), ds.getId());
    FeedDestination dest = feed.getDestination(ds.getId());
    DataOperation op = this.provider.beginOperation(dest, new DateTime());
    Dataset set = this.provider.createDataset(ds, Paths.get("a.txt"), Paths.get("b.txt"));
    op = this.provider.completeOperation(op.getId(), "", set);
    assertThat(op).isNotNull();
    assertThat(op.getState()).isEqualTo(State.SUCCESS);
}

Example 5 with Dataset

use of com.thinkbiganalytics.metadata.rest.model.op.Dataset in project kylo by Teradata.

the class MetadataClientTest method testCheckPrecondition.

// @Test
public void testCheckPrecondition() throws ParseException {
    Feed feedA = buildFeed("category", "feedA").post();
    Feed feedB = buildFeed("category", "feedB", "category", "feedA").post();
    HiveTableDatasource dsA = buildHiveTableDatasource("test-table").post();
    feedA = client.addDestination(feedA.getId(), dsA.getId());
    String destIdA = feedA.getDestinations().iterator().next().getId();
    DataOperation op = client.beginOperation(destIdA, "");
    op.setState(State.SUCCESS);
    op.setDataset(new Dataset(new DateTime(), dsA, ChangeType.UPDATE, ContentType.PARTITIONS));
    op = client.updateDataOperation(op);
    ServiceLevelAssessment assmt = client.assessPrecondition(feedB.getId());
    assertThat(assmt).isNotNull();
}