Examples with DirectoryDatasource com.thinkbiganalytics.metadata.rest.model.data.DirectoryDatasource used on opensource projects

Example 1 with DirectoryDatasource

use of com.thinkbiganalytics.metadata.rest.model.data.DirectoryDatasource in project kylo by Teradata.

the class MetadataClientTest method testBuildDirectoryDatasource.

// @Test
public void testBuildDirectoryDatasource() {
    DirectoryDatasource ds = buildDirectoryDatasource("test-dir").post();
    assertThat(ds).isNotNull();
}

Example 2 with DirectoryDatasource

use of com.thinkbiganalytics.metadata.rest.model.data.DirectoryDatasource in project kylo by Teradata.

the class MetadataClientProviderTest method testEnsureDirectoryDatasource.

@Test
public void testEnsureDirectoryDatasource() {
    this.provider.ensureDirectoryDatasource("test6", "", Paths.get("aaa", "bbb"));
    Datasource ds = this.provider.getDatasourceByName("test6");
    assertThat(ds).isNotNull();
    assertThat(ds).isInstanceOf(DirectoryDatasource.class);
    String dsId = ds.getId();
    DirectoryDatasource dds = (DirectoryDatasource) ds;
    assertThat(dds.getPath()).contains("aaa/bbb");
    ds = this.provider.ensureDirectoryDatasource("test6", "", Paths.get("aaa", "bbb"));
    assertThat(ds).isNotNull();
    assertThat(ds.getId()).isEqualTo(dsId);
}

Example 3 with DirectoryDatasource

use of com.thinkbiganalytics.metadata.rest.model.data.DirectoryDatasource in project kylo by Teradata.

the class MetadataClientProviderTest method testCompleteOperation.

@Test
public void testCompleteOperation() {
    Feed feed = this.provider.ensureFeed("category", "test10", "");
    DirectoryDatasource ds = this.provider.ensureDirectoryDatasource("test10", "", Paths.get("aaa", "bbb"));
    feed = this.provider.ensureFeedDestination(feed.getId(), ds.getId());
    FeedDestination dest = feed.getDestination(ds.getId());
    DataOperation op = this.provider.beginOperation(dest, new DateTime());
    Dataset set = this.provider.createDataset(ds, Paths.get("a.txt"), Paths.get("b.txt"));
    op = this.provider.completeOperation(op.getId(), "", set);
    assertThat(op).isNotNull();
    assertThat(op.getState()).isEqualTo(State.SUCCESS);
}

Example 4 with DirectoryDatasource

use of com.thinkbiganalytics.metadata.rest.model.data.DirectoryDatasource in project kylo by Teradata.

the class TerminateDirectoryFeed method completeOperation.

@Override
protected DataOperation completeOperation(ProcessContext context, FlowFile flowFile, Datasource datasource, DataOperation op, DataOperation.State state) {
    MetadataProvider provider = getProviderService(context).getProvider();
    DirectoryDatasource dds = (DirectoryDatasource) datasource;
    if (state == State.SUCCESS) {
        ArrayList<Path> paths = new ArrayList<>();
        // TODO Extract file paths from flow file
        Dataset dataset = provider.createDataset(dds, paths);
        return provider.completeOperation(op.getId(), "", dataset);
    } else {
        return provider.completeOperation(op.getId(), "", state);
    }
}