Examples with DatasetInstanceConfiguration io.cdap.cdap.proto.DatasetInstanceConfiguration used on opensource projects

Example 6 with DatasetInstanceConfiguration

use of io.cdap.cdap.proto.DatasetInstanceConfiguration in project cdap by caskdata.

the class MetadataHttpHandlerTestRun method testSearchResultSorting.

@Test
public void testSearchResultSorting() throws Exception {
    NamespaceId namespace = new NamespaceId("sorting");
    namespaceClient.create(new NamespaceMeta.Builder().setName(namespace).build());
    DatasetId text = namespace.dataset("text");
    DatasetId dataset = namespace.dataset("mydataset");
    DatasetId view = namespace.dataset("view");
    // create entities so system metadata is annotated
    // also ensure that they are created at least 1 ms apart
    DatasetInstanceConfiguration instanceConfig = new DatasetInstanceConfiguration(Table.class.getName(), Collections.emptyMap());
    datasetClient.create(text, instanceConfig);
    TimeUnit.MILLISECONDS.sleep(1);
    datasetClient.create(view, instanceConfig);
    TimeUnit.MILLISECONDS.sleep(1);
    datasetClient.create(dataset, instanceConfig);
    // search with bad sort param
    Set<String> targets = Collections.emptySet();
    // test ascending order of entity name
    waitForInOrder(() -> searchMetadata(namespace, "*", targets, MetadataConstants.ENTITY_NAME_KEY + " asc"), dataset, text, view);
    // test descending order of entity name
    assertInOrder(searchMetadata(namespace, "*", targets, MetadataConstants.ENTITY_NAME_KEY + " desc"), view, text, dataset);
    // test ascending order of creation time
    assertInOrder(searchMetadata(namespace, "*", targets, MetadataConstants.CREATION_TIME_KEY + " asc"), text, view, dataset);
    // test descending order of creation time
    assertInOrder(searchMetadata(namespace, "*", targets, MetadataConstants.CREATION_TIME_KEY + " desc"), dataset, view, text);
    // cleanup
    namespaceClient.delete(namespace);
}

Example 7 with DatasetInstanceConfiguration

use of io.cdap.cdap.proto.DatasetInstanceConfiguration in project cdap by caskdata.

the class MetadataHttpHandlerTestRun method testSearchResultPagination.

@Test
public void testSearchResultPagination() throws Exception {
    NamespaceId namespace = new NamespaceId("pagination");
    namespaceClient.create(new NamespaceMeta.Builder().setName(namespace).build());
    DatasetId text = namespace.dataset("text");
    DatasetId dataset = namespace.dataset("mydataset");
    DatasetId view = namespace.dataset("view");
    DatasetId tracker = namespace.dataset("_auditLog");
    // create entities so system metadata is annotated
    DatasetInstanceConfiguration instanceConfig = new DatasetInstanceConfiguration(Table.class.getName(), Collections.emptyMap());
    datasetClient.create(text, instanceConfig);
    datasetClient.create(view, instanceConfig);
    datasetClient.create(dataset, instanceConfig);
    datasetClient.create(tracker, instanceConfig);
    // search with showHidden to true
    Set<String> targets = Collections.emptySet();
    String sort = MetadataConstants.ENTITY_NAME_KEY + " asc";
    // search to get all the above entities offset 0, limit integer max  and cursors 0
    MetadataSearchResponse response = waitForInOrder(() -> searchMetadata(namespace, "*", targets, sort, 0, Integer.MAX_VALUE, 0, null, true), tracker, dataset, text, view);
    Assert.assertTrue(response.getCursors().isEmpty());
    // no offset, limit 1, no cursors
    response = assertInOrder(searchMetadata(namespace, "*", targets, sort, 0, 1, 0, null), dataset);
    Assert.assertEquals(0, response.getNumCursors());
    Assert.assertTrue(response.getCursors().isEmpty());
    // no offset, limit 1, 2 cursors, should return 1st result, with 2 cursors
    response = assertInOrder(searchMetadata(namespace, "*", targets, sort, 0, 1, 2, null), dataset);
    Assert.assertEquals(1, response.getNumCursors());
    Assert.assertEquals(1, response.getCursors().size());
    // offset 1, limit 1, 2 cursors, should return 2nd result, with only 1 cursor since we don't have enough data
    response = assertInOrder(searchMetadata(namespace, "*", targets, sort, 1, 1, 2, null), text);
    Assert.assertEquals(1, response.getNumCursors());
    Assert.assertEquals(1, response.getCursors().size());
    // offset 2, limit 1, 2 cursors, should return 3rd result, with 0 cursors since we don't have enough data
    response = assertInOrder(searchMetadata(namespace, "*", targets, sort, 2, 1, 2, null), view);
    Assert.assertEquals(0, response.getNumCursors());
    Assert.assertTrue(response.getCursors().isEmpty());
    // offset 3, limit 1, 2 cursors, should 0 results, with 0 cursors since we don't have enough data
    response = assertSearch(searchMetadata(namespace, "*", targets, sort, 3, 1, 2, null));
    Assert.assertEquals(0, response.getNumCursors());
    Assert.assertTrue(response.getCursors().isEmpty());
    // no offset, no limit, should return everything
    response = assertInOrder(searchMetadata(namespace, "*", targets, sort, 0, Integer.MAX_VALUE, 4, null), dataset, text, view);
    Assert.assertEquals(0, response.getNumCursors());
    Assert.assertTrue(response.getCursors().isEmpty());
    // cleanup
    namespaceClient.delete(namespace);
}

Example 8 with DatasetInstanceConfiguration

use of io.cdap.cdap.proto.DatasetInstanceConfiguration in project cdap by caskdata.

the class DatasetClientTestRun method testAll.

@Test
public void testAll() throws Exception {
    DatasetModuleId module = TEST_NAMESPACE.datasetModule(StandaloneDatasetModule.NAME);
    DatasetTypeId type = TEST_NAMESPACE.datasetType(StandaloneDataset.class.getName());
    DatasetModuleId moduleInOtherNamespace = OTHER_NAMESPACE.datasetModule(StandaloneDatasetModule.NAME);
    DatasetTypeId typeInOtherNamespace = OTHER_NAMESPACE.datasetType(StandaloneDataset.class.getName());
    int numBaseModules = moduleClient.list(TEST_NAMESPACE).size();
    int numBaseTypes = typeClient.list(TEST_NAMESPACE).size();
    LOG.info("Adding Dataset module");
    File moduleJarFile = createAppJarFile(StandaloneDatasetModule.class);
    moduleClient.add(TEST_NAMESPACE.datasetModule(StandaloneDatasetModule.NAME), StandaloneDatasetModule.class.getName(), moduleJarFile);
    Assert.assertEquals(numBaseModules + 1, moduleClient.list(TEST_NAMESPACE).size());
    Assert.assertEquals(numBaseTypes + 2, typeClient.list(TEST_NAMESPACE).size());
    LOG.info("Checking that the new Dataset module exists");
    DatasetModuleMeta datasetModuleMeta = moduleClient.get(module);
    Assert.assertNotNull(datasetModuleMeta);
    Assert.assertEquals(StandaloneDatasetModule.NAME, datasetModuleMeta.getName());
    LOG.info("Checking that the new Dataset module does not exist in a different namespace");
    try {
        moduleClient.get(moduleInOtherNamespace);
        Assert.fail("datasetModule found in namespace other than one in which it was expected");
    } catch (DatasetModuleNotFoundException expected) {
    // expected
    }
    LOG.info("Checking that the new Dataset type exists");
    DatasetTypeMeta datasetTypeMeta = typeClient.get(type);
    Assert.assertNotNull(datasetTypeMeta);
    Assert.assertEquals(type.getType(), datasetTypeMeta.getName());
    datasetTypeMeta = typeClient.get(type);
    Assert.assertNotNull(datasetTypeMeta);
    Assert.assertEquals(StandaloneDataset.class.getName(), datasetTypeMeta.getName());
    LOG.info("Checking that the new Dataset type does not exist in a different namespace");
    try {
        typeClient.get(typeInOtherNamespace);
        Assert.fail("datasetType found in namespace other than one in which it was expected");
    } catch (DatasetTypeNotFoundException expected) {
    // expected
    }
    LOG.info("Creating, truncating, and deleting dataset of new Dataset type");
    // Before creating dataset, there are some system datasets already exist
    int numBaseDataset = datasetClient.list(TEST_NAMESPACE).size();
    DatasetId instance = TEST_NAMESPACE.dataset("testDataset");
    String description = "test description";
    datasetClient.create(instance, new DatasetInstanceConfiguration(StandaloneDataset.TYPE_NAME, Collections.<String, String>emptyMap(), description, null));
    Assert.assertEquals(numBaseDataset + 1, datasetClient.list(TEST_NAMESPACE).size());
    // Assert dataset summary for the newly created dataset
    DatasetSpecificationSummary expectedSpec = new DatasetSpecificationSummary(instance.getDataset(), StandaloneDataset.TYPE_NAME, description, Collections.<String, String>emptyMap());
    Assert.assertEquals(expectedSpec, getSpecForDataset(instance, datasetClient.list(TEST_NAMESPACE)));
    datasetClient.truncate(instance);
    DatasetMeta metaBefore = datasetClient.get(instance);
    Assert.assertEquals(0, metaBefore.getSpec().getProperties().size());
    datasetClient.update(instance, ImmutableMap.of("sdf", "foo", "abc", "123"));
    DatasetMeta metaAfter = datasetClient.get(instance);
    Assert.assertEquals(2, metaAfter.getSpec().getProperties().size());
    Assert.assertTrue(metaAfter.getSpec().getProperties().containsKey("sdf"));
    Assert.assertTrue(metaAfter.getSpec().getProperties().containsKey("abc"));
    Assert.assertEquals("foo", metaAfter.getSpec().getProperties().get("sdf"));
    Assert.assertEquals("123", metaAfter.getSpec().getProperties().get("abc"));
    datasetClient.updateExisting(instance, ImmutableMap.of("sdf", "fzz"));
    metaAfter = datasetClient.get(instance);
    Assert.assertEquals(2, metaAfter.getSpec().getProperties().size());
    Assert.assertTrue(metaAfter.getSpec().getProperties().containsKey("sdf"));
    Assert.assertTrue(metaAfter.getSpec().getProperties().containsKey("abc"));
    Assert.assertEquals("fzz", metaAfter.getSpec().getProperties().get("sdf"));
    Assert.assertEquals("123", metaAfter.getSpec().getProperties().get("abc"));
    datasetClient.delete(instance);
    Assert.assertEquals(numBaseDataset, datasetClient.list(TEST_NAMESPACE).size());
    LOG.info("Creating and deleting multiple Datasets");
    for (int i = 1; i <= 3; i++) {
        datasetClient.create(TEST_NAMESPACE.dataset("testDataset" + i), StandaloneDataset.TYPE_NAME);
    }
    Assert.assertEquals(numBaseDataset + 3, datasetClient.list(TEST_NAMESPACE).size());
    for (int i = 1; i <= 3; i++) {
        datasetClient.delete(TEST_NAMESPACE.dataset("testDataset" + i));
    }
    Assert.assertEquals(numBaseDataset, datasetClient.list(TEST_NAMESPACE).size());
    LOG.info("Deleting Dataset module");
    moduleClient.delete(module);
    Assert.assertEquals(numBaseModules, moduleClient.list(TEST_NAMESPACE).size());
    Assert.assertEquals(numBaseTypes, typeClient.list(TEST_NAMESPACE).size());
    LOG.info("Adding Dataset module and then deleting all Dataset modules");
    moduleClient.add(TEST_NAMESPACE.datasetModule("testModule1"), StandaloneDatasetModule.class.getName(), moduleJarFile);
    Assert.assertEquals(numBaseModules + 1, moduleClient.list(TEST_NAMESPACE).size());
    Assert.assertEquals(numBaseTypes + 2, typeClient.list(TEST_NAMESPACE).size());
    moduleClient.deleteAll(TEST_NAMESPACE);
    Assert.assertEquals(numBaseModules, moduleClient.list(TEST_NAMESPACE).size());
    Assert.assertEquals(numBaseTypes, typeClient.list(TEST_NAMESPACE).size());
}

Example 9 with DatasetInstanceConfiguration

use of io.cdap.cdap.proto.DatasetInstanceConfiguration in project cdap by caskdata.

the class MetadataHttpHandlerTestRun method testSearchResultPaginationWithTargetType.

@Test
public void testSearchResultPaginationWithTargetType() throws Exception {
    // note that the ordering of the entity creations and the sort param used in this test case matter, in order to
    // reproduce the scenario that caused the issue CDAP-7881
    NamespaceId namespace = new NamespaceId("pagination_with_target_type");
    namespaceClient.create(new NamespaceMeta.Builder().setName(namespace).build());
    DatasetId trackerDataset = namespace.dataset("_auditLog");
    DatasetId mydataset = namespace.dataset("mydataset");
    // the creation order below will determine how we see the entities in search result sorted by entity creation time
    // in ascending order
    datasetClient.create(trackerDataset, new DatasetInstanceConfiguration(Table.class.getName(), Collections.emptyMap()));
    datasetClient.create(mydataset, new DatasetInstanceConfiguration(Table.class.getName(), Collections.emptyMap()));
    // do sorting with creation time here since the testSearchResultPagination does with entity name
    // the sorted result order _auditLog mydataset text2 text1 (ascending: creation from earliest time)
    String sort = MetadataConstants.CREATION_TIME_KEY + " " + SortInfo.SortOrder.ASC;
    // offset 1, limit 2, 2 cursors, should return just the dataset created above other than trackerDataset even
    // though it was created before since showHidden is false and it should not affect pagination
    MetadataSearchResponse response = waitForInOrder(() -> searchMetadata(namespace, "*", ImmutableSet.of(MetadataEntity.DATASET), sort, 0, 2, 2, null), mydataset);
    Assert.assertTrue(response.getCursors().isEmpty());
}

Example 10 with DatasetInstanceConfiguration

use of io.cdap.cdap.proto.DatasetInstanceConfiguration in project cdap by caskdata.

the class IntegrationTestManager method addDatasetInstance.

@Override
public <T extends DatasetAdmin> T addDatasetInstance(String datasetType, DatasetId datasetId, DatasetProperties props) throws Exception {
    DatasetInstanceConfiguration dsConf = new DatasetInstanceConfiguration(datasetType, props.getProperties(), props.getDescription(), null);
    datasetClient.create(datasetId, dsConf);
    return (T) new RemoteDatasetAdmin(datasetClient, datasetId, dsConf);
}