Examples with DatasetSpecificationSummary io.cdap.cdap.proto.DatasetSpecificationSummary used on opensource projects

Example 16 with DatasetSpecificationSummary

use of io.cdap.cdap.proto.DatasetSpecificationSummary in project cdap by cdapio.

the class AbstractDatasetFrameworkTest method testNamespaceInstanceIsolation.

@Test
@SuppressWarnings("ConstantConditions")
public void testNamespaceInstanceIsolation() throws Exception {
    DatasetFramework framework = getFramework();
    // create 2 namespaces
    NamespaceId namespace1 = new NamespaceId("ns1");
    NamespaceId namespace2 = new NamespaceId("ns2");
    namespaceAdmin.create(new NamespaceMeta.Builder().setName(namespace1).build());
    namespaceAdmin.create(new NamespaceMeta.Builder().setName(namespace2).build());
    namespacePathLocator.get(namespace1).mkdirs();
    namespacePathLocator.get(namespace2).mkdirs();
    // create 2 tables, one in each namespace. both tables have the same name.
    DatasetId table1ID = namespace1.dataset("table");
    DatasetId table2ID = namespace2.dataset("table");
    // have slightly different properties so that we can distinguish between them
    framework.addInstance(Table.class.getName(), table1ID, DatasetProperties.builder().add("tag", "table1").build());
    framework.addInstance(Table.class.getName(), table2ID, DatasetProperties.builder().add("tag", "table2").build());
    // perform some data operations to make sure they are not the same underlying table
    final Table table1 = framework.getDataset(table1ID, Maps.<String, String>newHashMap(), null);
    final Table table2 = framework.getDataset(table2ID, Maps.<String, String>newHashMap(), null);
    TransactionExecutor txnl = new DefaultTransactionExecutor(new MinimalTxSystemClient(), (TransactionAware) table1, (TransactionAware) table2);
    txnl.execute(new TransactionExecutor.Subroutine() {

        @Override
        public void apply() throws Exception {
            table1.put(Bytes.toBytes("rowkey"), Bytes.toBytes("column"), Bytes.toBytes("val1"));
            table2.put(Bytes.toBytes("rowkey"), Bytes.toBytes("column"), Bytes.toBytes("val2"));
        }
    });
    // check data is different, which means they are different underlying tables
    txnl.execute(new TransactionExecutor.Subroutine() {

        @Override
        public void apply() throws Exception {
            Assert.assertEquals("val1", Bytes.toString(table1.get(Bytes.toBytes("rowkey"), Bytes.toBytes("column"))));
            Assert.assertEquals("val2", Bytes.toString(table2.get(Bytes.toBytes("rowkey"), Bytes.toBytes("column"))));
        }
    });
    // check get all in a namespace only includes those in that namespace
    Collection<DatasetSpecificationSummary> specs = framework.getInstances(namespace1);
    Assert.assertEquals(1, specs.size());
    Assert.assertEquals("table1", specs.iterator().next().getProperties().get("tag"));
    specs = framework.getInstances(namespace2);
    Assert.assertEquals(1, specs.size());
    Assert.assertEquals("table2", specs.iterator().next().getProperties().get("tag"));
    // delete one instance and make sure the other still exists
    framework.deleteInstance(table1ID);
    Assert.assertFalse(framework.hasInstance(table1ID));
    Assert.assertTrue(framework.hasInstance(table2ID));
    // delete all instances in one namespace and make sure the other still exists
    framework.addInstance(Table.class.getName(), table1ID, DatasetProperties.EMPTY);
    framework.deleteAllInstances(namespace1);
    Assert.assertTrue(framework.hasInstance(table2ID));
    // delete one namespace and make sure the other still exists
    namespacePathLocator.get(namespace1).delete(true);
    Assert.assertTrue(framework.hasInstance(table2ID));
}

Example 17 with DatasetSpecificationSummary

use of io.cdap.cdap.proto.DatasetSpecificationSummary in project cdap by cdapio.

the class DatasetInstanceHandlerTest method testBasics.

@Test
public void testBasics() throws Exception {
    // nothing has been created, modules and types list is empty
    List<DatasetSpecificationSummary> instances = getInstances().getResponseObject();
    // nothing in the beginning
    Assert.assertEquals(0, instances.size());
    try {
        // create dataset instance with type that is not yet known to the system should fail
        DatasetProperties props = DatasetProperties.builder().add("prop1", "val1").build();
        Assert.assertEquals(HttpStatus.SC_NOT_FOUND, createInstance("dataset1", "datasetType2", props).getResponseCode());
        // deploy modules
        deployModule("module1", TestModule1.class);
        deployModule("module2", TestModule2.class);
        // create dataset instance
        String description = "test instance description";
        HttpResponse response = createInstance("dataset1", "datasetType2", description, props);
        Assert.assertEquals(HttpStatus.SC_OK, response.getResponseCode());
        // verify module cannot be deleted which type is used for the dataset
        int modulesBeforeDelete = getModules().getResponseObject().size();
        Assert.assertEquals(HttpStatus.SC_CONFLICT, deleteModule("module2").getResponseCode());
        Assert.assertEquals(HttpStatus.SC_CONFLICT, deleteModules().getResponseCode());
        Assert.assertEquals(modulesBeforeDelete, getModules().getResponseObject().size());
        // Verify that dataset instance can be retrieved using specified properties
        Map<String, String> properties = new HashMap<>(props.getProperties());
        instances = getInstancesWithProperties(NamespaceId.DEFAULT.getNamespace(), properties).getResponseObject();
        Assert.assertEquals(1, instances.size());
        properties.put("some_prop_not_associated_with_dataset", "somevalue");
        instances = getInstancesWithProperties(NamespaceId.DEFAULT.getNamespace(), properties).getResponseObject();
        Assert.assertEquals(0, instances.size());
        // verify instance was created
        instances = getInstances().getResponseObject();
        Assert.assertEquals(1, instances.size());
        // verifying spec is same as expected
        DatasetSpecification dataset1Spec = createType2Spec("dataset1", "datasetType2", description, props);
        Assert.assertEquals(spec2Summary(dataset1Spec), instances.get(0));
        // verify created instance info can be retrieved
        DatasetMeta datasetInfo = getInstanceObject("dataset1").getResponseObject();
        Assert.assertEquals(dataset1Spec, datasetInfo.getSpec());
        Assert.assertEquals(dataset1Spec.getType(), datasetInfo.getType().getName());
        // type meta should have 2 modules that has to be loaded to create type's class
        // and in the order they must be loaded
        List<DatasetModuleMeta> modules = datasetInfo.getType().getModules();
        Assert.assertEquals(2, modules.size());
        DatasetTypeHandlerTest.verify(modules.get(0), "module1", TestModule1.class, ImmutableList.of("datasetType1"), Collections.emptyList(), ImmutableList.of("module2"));
        DatasetTypeHandlerTest.verify(modules.get(1), "module2", TestModule2.class, ImmutableList.of("datasetType2"), ImmutableList.of("module1"), Collections.emptyList());
        // try to retrieve non-existed instance
        Assert.assertEquals(HttpStatus.SC_NOT_FOUND, getInstance("non-existing-dataset").getResponseCode());
        // cannot create instance with same name again
        Assert.assertEquals(HttpStatus.SC_CONFLICT, createInstance("dataset1", "datasetType2", props).getResponseCode());
        Assert.assertEquals(1, getInstances().getResponseObject().size());
        // cannot delete non-existing dataset instance
        Assert.assertEquals(HttpStatus.SC_NOT_FOUND, deleteInstance("non-existing-dataset").getResponseCode());
        Assert.assertEquals(1, getInstances().getResponseObject().size());
        // verify creation of dataset instance with null properties
        Assert.assertEquals(HttpStatus.SC_OK, createInstance("nullPropertiesTable", "datasetType2").getResponseCode());
        // since dataset instance description is not provided, we are using the description given by the dataset type
        DatasetSpecification nullPropertiesTableSpec = createType2Spec("nullPropertiesTable", "datasetType2", TestModule2.DESCRIPTION, DatasetProperties.EMPTY);
        DatasetSpecificationSummary actualSummary = getSummaryForInstance("nullPropertiesTable", getInstances().getResponseObject());
        Assert.assertEquals(spec2Summary(nullPropertiesTableSpec), actualSummary);
        // delete dataset instance
        Assert.assertEquals(HttpStatus.SC_OK, deleteInstance("dataset1").getResponseCode());
        Assert.assertEquals(HttpStatus.SC_OK, deleteInstance("nullPropertiesTable").getResponseCode());
        Assert.assertEquals(0, getInstances().getResponseObject().size());
        // create workflow local dataset instance
        DatasetProperties localDSProperties = DatasetProperties.builder().add("prop1", "val1").add(Constants.AppFabric.WORKFLOW_LOCAL_DATASET_PROPERTY, "true").build();
        Assert.assertEquals(HttpStatus.SC_OK, createInstance("localDSInstance", "datasetType2", localDSProperties).getResponseCode());
        // getInstances call should still return 0
        Assert.assertEquals(0, getInstances().getResponseObject().size());
        Assert.assertEquals(HttpStatus.SC_OK, deleteInstance("localDSInstance").getResponseCode());
        // delete dataset modules
        Assert.assertEquals(HttpStatus.SC_OK, deleteModule("module2").getResponseCode());
        Assert.assertEquals(HttpStatus.SC_OK, deleteModule("module1").getResponseCode());
    } finally {
        deleteInstance("dataset1");
        deleteInstance("nullPropertiesTable");
        deleteInstance("localDSInstance");
        deleteModule("module2");
        deleteModule("module1");
    }
}

Example 18 with DatasetSpecificationSummary

use of io.cdap.cdap.proto.DatasetSpecificationSummary in project cdap by cdapio.

the class IntegrationTestBase method assertNoUserDatasets.

private void assertNoUserDatasets(NamespaceId namespace) throws Exception {
    DatasetClient datasetClient = getDatasetClient();
    List<DatasetSpecificationSummary> datasets = datasetClient.list(namespace);
    Iterable<DatasetSpecificationSummary> userDatasets = Iterables.filter(datasets, new Predicate<DatasetSpecificationSummary>() {

        @Override
        public boolean apply(DatasetSpecificationSummary input) {
            return isUserDataset(input);
        }
    });
    Iterable<String> userDatasetNames = Iterables.transform(userDatasets, new Function<DatasetSpecificationSummary, String>() {

        @Override
        public String apply(DatasetSpecificationSummary input) {
            return input.getName();
        }
    });
    Assert.assertFalse("Must have no user datasets, but found the following user datasets: " + Joiner.on(", ").join(userDatasetNames), userDatasets.iterator().hasNext());
}

Example 19 with DatasetSpecificationSummary

use of io.cdap.cdap.proto.DatasetSpecificationSummary in project cdap by caskdata.

the class ListDatasetInstancesCommand method perform.

@Override
public void perform(Arguments arguments, PrintStream output) throws Exception {
    List<DatasetSpecificationSummary> datasetMetas = datasetClient.list(cliConfig.getCurrentNamespace());
    Table table = Table.builder().setHeader("name", "type", "description").setRows(datasetMetas, new RowMaker<DatasetSpecificationSummary>() {

        @Override
        public List<?> makeRow(DatasetSpecificationSummary object) {
            return Lists.newArrayList(object.getName(), object.getType(), object.getDescription());
        }
    }).build();
    cliConfig.getTableRenderer().render(cliConfig, output, table);
}

Example 20 with DatasetSpecificationSummary

use of io.cdap.cdap.proto.DatasetSpecificationSummary in project cdap by caskdata.

the class DatasetClientTestRun method testAll.

@Test
public void testAll() throws Exception {
    DatasetModuleId module = TEST_NAMESPACE.datasetModule(StandaloneDatasetModule.NAME);
    DatasetTypeId type = TEST_NAMESPACE.datasetType(StandaloneDataset.class.getName());
    DatasetModuleId moduleInOtherNamespace = OTHER_NAMESPACE.datasetModule(StandaloneDatasetModule.NAME);
    DatasetTypeId typeInOtherNamespace = OTHER_NAMESPACE.datasetType(StandaloneDataset.class.getName());
    int numBaseModules = moduleClient.list(TEST_NAMESPACE).size();
    int numBaseTypes = typeClient.list(TEST_NAMESPACE).size();
    LOG.info("Adding Dataset module");
    File moduleJarFile = createAppJarFile(StandaloneDatasetModule.class);
    moduleClient.add(TEST_NAMESPACE.datasetModule(StandaloneDatasetModule.NAME), StandaloneDatasetModule.class.getName(), moduleJarFile);
    Assert.assertEquals(numBaseModules + 1, moduleClient.list(TEST_NAMESPACE).size());
    Assert.assertEquals(numBaseTypes + 2, typeClient.list(TEST_NAMESPACE).size());
    LOG.info("Checking that the new Dataset module exists");
    DatasetModuleMeta datasetModuleMeta = moduleClient.get(module);
    Assert.assertNotNull(datasetModuleMeta);
    Assert.assertEquals(StandaloneDatasetModule.NAME, datasetModuleMeta.getName());
    LOG.info("Checking that the new Dataset module does not exist in a different namespace");
    try {
        moduleClient.get(moduleInOtherNamespace);
        Assert.fail("datasetModule found in namespace other than one in which it was expected");
    } catch (DatasetModuleNotFoundException expected) {
    // expected
    }
    LOG.info("Checking that the new Dataset type exists");
    DatasetTypeMeta datasetTypeMeta = typeClient.get(type);
    Assert.assertNotNull(datasetTypeMeta);
    Assert.assertEquals(type.getType(), datasetTypeMeta.getName());
    datasetTypeMeta = typeClient.get(type);
    Assert.assertNotNull(datasetTypeMeta);
    Assert.assertEquals(StandaloneDataset.class.getName(), datasetTypeMeta.getName());
    LOG.info("Checking that the new Dataset type does not exist in a different namespace");
    try {
        typeClient.get(typeInOtherNamespace);
        Assert.fail("datasetType found in namespace other than one in which it was expected");
    } catch (DatasetTypeNotFoundException expected) {
    // expected
    }
    LOG.info("Creating, truncating, and deleting dataset of new Dataset type");
    // Before creating dataset, there are some system datasets already exist
    int numBaseDataset = datasetClient.list(TEST_NAMESPACE).size();
    DatasetId instance = TEST_NAMESPACE.dataset("testDataset");
    String description = "test description";
    datasetClient.create(instance, new DatasetInstanceConfiguration(StandaloneDataset.TYPE_NAME, Collections.<String, String>emptyMap(), description, null));
    Assert.assertEquals(numBaseDataset + 1, datasetClient.list(TEST_NAMESPACE).size());
    // Assert dataset summary for the newly created dataset
    DatasetSpecificationSummary expectedSpec = new DatasetSpecificationSummary(instance.getDataset(), StandaloneDataset.TYPE_NAME, description, Collections.<String, String>emptyMap());
    Assert.assertEquals(expectedSpec, getSpecForDataset(instance, datasetClient.list(TEST_NAMESPACE)));
    datasetClient.truncate(instance);
    DatasetMeta metaBefore = datasetClient.get(instance);
    Assert.assertEquals(0, metaBefore.getSpec().getProperties().size());
    datasetClient.update(instance, ImmutableMap.of("sdf", "foo", "abc", "123"));
    DatasetMeta metaAfter = datasetClient.get(instance);
    Assert.assertEquals(2, metaAfter.getSpec().getProperties().size());
    Assert.assertTrue(metaAfter.getSpec().getProperties().containsKey("sdf"));
    Assert.assertTrue(metaAfter.getSpec().getProperties().containsKey("abc"));
    Assert.assertEquals("foo", metaAfter.getSpec().getProperties().get("sdf"));
    Assert.assertEquals("123", metaAfter.getSpec().getProperties().get("abc"));
    datasetClient.updateExisting(instance, ImmutableMap.of("sdf", "fzz"));
    metaAfter = datasetClient.get(instance);
    Assert.assertEquals(2, metaAfter.getSpec().getProperties().size());
    Assert.assertTrue(metaAfter.getSpec().getProperties().containsKey("sdf"));
    Assert.assertTrue(metaAfter.getSpec().getProperties().containsKey("abc"));
    Assert.assertEquals("fzz", metaAfter.getSpec().getProperties().get("sdf"));
    Assert.assertEquals("123", metaAfter.getSpec().getProperties().get("abc"));
    datasetClient.delete(instance);
    Assert.assertEquals(numBaseDataset, datasetClient.list(TEST_NAMESPACE).size());
    LOG.info("Creating and deleting multiple Datasets");
    for (int i = 1; i <= 3; i++) {
        datasetClient.create(TEST_NAMESPACE.dataset("testDataset" + i), StandaloneDataset.TYPE_NAME);
    }
    Assert.assertEquals(numBaseDataset + 3, datasetClient.list(TEST_NAMESPACE).size());
    for (int i = 1; i <= 3; i++) {
        datasetClient.delete(TEST_NAMESPACE.dataset("testDataset" + i));
    }
    Assert.assertEquals(numBaseDataset, datasetClient.list(TEST_NAMESPACE).size());
    LOG.info("Deleting Dataset module");
    moduleClient.delete(module);
    Assert.assertEquals(numBaseModules, moduleClient.list(TEST_NAMESPACE).size());
    Assert.assertEquals(numBaseTypes, typeClient.list(TEST_NAMESPACE).size());
    LOG.info("Adding Dataset module and then deleting all Dataset modules");
    moduleClient.add(TEST_NAMESPACE.datasetModule("testModule1"), StandaloneDatasetModule.class.getName(), moduleJarFile);
    Assert.assertEquals(numBaseModules + 1, moduleClient.list(TEST_NAMESPACE).size());
    Assert.assertEquals(numBaseTypes + 2, typeClient.list(TEST_NAMESPACE).size());
    moduleClient.deleteAll(TEST_NAMESPACE);
    Assert.assertEquals(numBaseModules, moduleClient.list(TEST_NAMESPACE).size());
    Assert.assertEquals(numBaseTypes, typeClient.list(TEST_NAMESPACE).size());
}