Examples with DatasetProperties co.cask.cdap.api.dataset.DatasetProperties used on opensource projects

Example 26 with DatasetProperties

use of co.cask.cdap.api.dataset.DatasetProperties in project cdap by caskdata.

the class DatasetInstanceHandlerTest method testBasics.

@Test
public void testBasics() throws Exception {
    // nothing has been created, modules and types list is empty
    List<DatasetSpecificationSummary> instances = getInstances().getResponseObject();
    // nothing in the beginning
    Assert.assertEquals(0, instances.size());
    try {
        // create dataset instance with type that is not yet known to the system should fail
        DatasetProperties props = DatasetProperties.builder().add("prop1", "val1").build();
        Assert.assertEquals(HttpStatus.SC_NOT_FOUND, createInstance("dataset1", "datasetType2", props).getResponseCode());
        // deploy modules
        deployModule("module1", TestModule1.class);
        deployModule("module2", TestModule2.class);
        // create dataset instance
        String description = "test instance description";
        HttpResponse response = createInstance("dataset1", "datasetType2", description, props);
        Assert.assertEquals(HttpStatus.SC_OK, response.getResponseCode());
        // verify module cannot be deleted which type is used for the dataset
        int modulesBeforeDelete = getModules().getResponseObject().size();
        Assert.assertEquals(HttpStatus.SC_CONFLICT, deleteModule("module2").getResponseCode());
        Assert.assertEquals(HttpStatus.SC_CONFLICT, deleteModules().getResponseCode());
        Assert.assertEquals(modulesBeforeDelete, getModules().getResponseObject().size());
        // Verify that dataset instance can be retrieved using specified properties
        Map<String, String> properties = new HashMap<>();
        properties.putAll(props.getProperties());
        instances = getInstancesWithProperties(NamespaceId.DEFAULT.getNamespace(), properties).getResponseObject();
        Assert.assertEquals(1, instances.size());
        properties.put("some_prop_not_associated_with_dataset", "somevalue");
        instances = getInstancesWithProperties(NamespaceId.DEFAULT.getNamespace(), properties).getResponseObject();
        Assert.assertEquals(0, instances.size());
        // verify instance was created
        instances = getInstances().getResponseObject();
        Assert.assertEquals(1, instances.size());
        // verifying spec is same as expected
        DatasetSpecification dataset1Spec = createType2Spec("dataset1", "datasetType2", description, props);
        Assert.assertEquals(spec2Summary(dataset1Spec), instances.get(0));
        // verify created instance info can be retrieved
        DatasetMeta datasetInfo = getInstanceObject("dataset1").getResponseObject();
        Assert.assertEquals(dataset1Spec, datasetInfo.getSpec());
        Assert.assertEquals(dataset1Spec.getType(), datasetInfo.getType().getName());
        // type meta should have 2 modules that has to be loaded to create type's class
        // and in the order they must be loaded
        List<DatasetModuleMeta> modules = datasetInfo.getType().getModules();
        Assert.assertEquals(2, modules.size());
        DatasetTypeHandlerTest.verify(modules.get(0), "module1", TestModule1.class, ImmutableList.of("datasetType1"), Collections.<String>emptyList(), ImmutableList.of("module2"));
        DatasetTypeHandlerTest.verify(modules.get(1), "module2", TestModule2.class, ImmutableList.of("datasetType2"), ImmutableList.of("module1"), Collections.<String>emptyList());
        // try to retrieve non-existed instance
        Assert.assertEquals(HttpStatus.SC_NOT_FOUND, getInstance("non-existing-dataset").getResponseCode());
        // cannot create instance with same name again
        Assert.assertEquals(HttpStatus.SC_CONFLICT, createInstance("dataset1", "datasetType2", props).getResponseCode());
        Assert.assertEquals(1, getInstances().getResponseObject().size());
        // cannot delete non-existing dataset instance
        Assert.assertEquals(HttpStatus.SC_NOT_FOUND, deleteInstance("non-existing-dataset").getResponseCode());
        Assert.assertEquals(1, getInstances().getResponseObject().size());
        // verify creation of dataset instance with null properties
        Assert.assertEquals(HttpStatus.SC_OK, createInstance("nullPropertiesTable", "datasetType2").getResponseCode());
        // since dataset instance description is not provided, we are using the description given by the dataset type
        DatasetSpecification nullPropertiesTableSpec = createType2Spec("nullPropertiesTable", "datasetType2", TestModule2.DESCRIPTION, DatasetProperties.EMPTY);
        DatasetSpecificationSummary actualSummary = getSummaryForInstance("nullPropertiesTable", getInstances().getResponseObject());
        Assert.assertEquals(spec2Summary(nullPropertiesTableSpec), actualSummary);
        // delete dataset instance
        Assert.assertEquals(HttpStatus.SC_OK, deleteInstance("dataset1").getResponseCode());
        Assert.assertEquals(HttpStatus.SC_OK, deleteInstance("nullPropertiesTable").getResponseCode());
        Assert.assertEquals(0, getInstances().getResponseObject().size());
        // create workflow local dataset instance
        DatasetProperties localDSProperties = DatasetProperties.builder().add("prop1", "val1").add(Constants.AppFabric.WORKFLOW_LOCAL_DATASET_PROPERTY, "true").build();
        Assert.assertEquals(HttpStatus.SC_OK, createInstance("localDSInstance", "datasetType2", localDSProperties).getResponseCode());
        // getInstances call should still return 0
        Assert.assertEquals(0, getInstances().getResponseObject().size());
        Assert.assertEquals(HttpStatus.SC_OK, deleteInstance("localDSInstance").getResponseCode());
        // delete dataset modules
        Assert.assertEquals(HttpStatus.SC_OK, deleteModule("module2").getResponseCode());
        Assert.assertEquals(HttpStatus.SC_OK, deleteModule("module1").getResponseCode());
    } finally {
        deleteInstance("dataset1");
        deleteInstance("nullPropertiesTable");
        deleteInstance("localDSInstance");
        deleteModule("module2");
        deleteModule("module1");
    }
}

Example 27 with DatasetProperties

use of co.cask.cdap.api.dataset.DatasetProperties in project cdap by caskdata.

the class DatasetDefinitionRegistryWithDefaultModules method testFileSetReconfigure.

private void testFileSetReconfigure(boolean expectSuccess, DatasetDefinition def, Boolean wasExternal, String path, Boolean newExternal, String newPath, DatasetProperties extraProps) throws IncompatibleUpdateException {
    Assert.assertTrue(def instanceof Reconfigurable);
    DatasetProperties props = buildFileSetProps(extraProps, wasExternal, path);
    DatasetProperties newProps = buildFileSetProps(extraProps, newExternal, newPath);
    DatasetSpecification spec = def.configure("fs", props);
    if (expectSuccess) {
        ((Reconfigurable) def).reconfigure("fs", newProps, spec);
    } else {
        try {
            ((Reconfigurable) def).reconfigure("fs", newProps, spec);
            Assert.fail("reconfigure should have thrown exception");
        } catch (IncompatibleUpdateException e) {
        // expected
        }
    }
}

Example 28 with DatasetProperties

use of co.cask.cdap.api.dataset.DatasetProperties in project cdap by caskdata.

the class DatasetDefinitionRegistryWithDefaultModules method testTimeSeriesReconfigure.

private void testTimeSeriesReconfigure(DatasetDefinition def) throws IncompatibleUpdateException {
    DatasetProperties props = DatasetProperties.builder().add(TimeseriesDataset.ATTR_TIME_INTERVAL_TO_STORE_PER_ROW, String.valueOf(TimeUnit.HOURS.toMillis(1))).build();
    DatasetProperties compatProps = TableProperties.builder().setTTL(TimeUnit.DAYS.toSeconds(1)).add(TimeseriesDataset.ATTR_TIME_INTERVAL_TO_STORE_PER_ROW, String.valueOf(TimeUnit.HOURS.toMillis(1))).build();
    DatasetProperties incompatProps = TableProperties.builder().setTTL(TimeUnit.DAYS.toSeconds(1)).add(TimeseriesDataset.ATTR_TIME_INTERVAL_TO_STORE_PER_ROW, String.valueOf(TimeUnit.HOURS.toMillis(2))).build();
    DatasetSpecification spec = def.configure("tt", props);
    Assert.assertTrue(def instanceof Reconfigurable);
    ((Reconfigurable) def).reconfigure("tt", compatProps, spec);
    try {
        ((Reconfigurable) def).reconfigure("tt", incompatProps, spec);
        Assert.fail("reconfigure should have thrown exception");
    } catch (IncompatibleUpdateException e) {
    // expected
    }
}

Example 29 with DatasetProperties

use of co.cask.cdap.api.dataset.DatasetProperties in project cdap by caskdata.

the class DatasetDefinitionRegistryWithDefaultModules method testPFSReconfigure.

@Test
public void testPFSReconfigure() throws IncompatibleUpdateException {
    DatasetDefinition pfsDef = registry.get(PartitionedFileSet.class.getName());
    Assert.assertTrue(pfsDef instanceof Reconfigurable);
    DatasetProperties props = PartitionedFileSetProperties.builder().setPartitioning(Partitioning.builder().addIntField("i").addStringField("s").build()).build();
    DatasetSpecification spec = pfsDef.configure("pfs", props);
    DatasetProperties noIprops = PartitionedFileSetProperties.builder().setPartitioning(Partitioning.builder().addStringField("s").build()).build();
    try {
        ((Reconfigurable) pfsDef).reconfigure("pfs", noIprops, spec);
        Assert.fail("reconfigure should have thrown exception");
    } catch (IncompatibleUpdateException e) {
    // expected
    }
    DatasetProperties longIprops = PartitionedFileSetProperties.builder().setPartitioning(Partitioning.builder().addLongField("i").addStringField("s").build()).build();
    try {
        ((Reconfigurable) pfsDef).reconfigure("pfs", longIprops, spec);
        Assert.fail("reconfigure should have thrown exception");
    } catch (IncompatibleUpdateException e) {
    // expected
    }
    DatasetProperties revProps = PartitionedFileSetProperties.builder().setPartitioning(Partitioning.builder().addStringField("s").addIntField("i").build()).build();
    try {
        ((Reconfigurable) pfsDef).reconfigure("pfs", revProps, spec);
        Assert.fail("reconfigure should have thrown exception");
    } catch (IncompatibleUpdateException e) {
    // expected
    }
    // test reconfiguring a PFS created before CDAP-13120, with no defaulted base path
    // no base path should be set for the reconfigured dataset either
    DatasetProperties oldProps = PartitionedFileSetProperties.builder().setPartitioning(Partitioning.builder().addStringField("s").build()).add(PartitionedFileSetDefinition.NAME_AS_BASE_PATH_DEFAULT, "false").build();
    DatasetSpecification oldSpec = pfsDef.configure("pfs", oldProps);
    DatasetSpecification newSpec = ((Reconfigurable) pfsDef).reconfigure("pfs", oldProps, oldSpec);
    // make sure base path is not set
    Assert.assertNull(newSpec.getSpecification("files").getProperty(FileSetProperties.BASE_PATH));
    // test reconfiguring a PFS created after CDAP-13120, where base path is default to the dataset name
    props = PartitionedFileSetProperties.builder().setPartitioning(Partitioning.builder().addStringField("s").build()).build();
    oldSpec = pfsDef.configure("pfs", props);
    newSpec = ((Reconfigurable) pfsDef).reconfigure("pfs", props, oldSpec);
    // make sure base path is similarly set, even when not explicitly given
    Assert.assertEquals("pfs", newSpec.getSpecification("files").getProperty(FileSetProperties.BASE_PATH));
    // make sure it is set for subsequent reconfigures as well
    newSpec = ((Reconfigurable) pfsDef).reconfigure("pfs", props, oldSpec);
    // make sure base path is similarly set, even when not explicitly given
    Assert.assertEquals("pfs", newSpec.getSpecification("files").getProperty(FileSetProperties.BASE_PATH));
}

Example 30 with DatasetProperties

use of co.cask.cdap.api.dataset.DatasetProperties in project cdap by caskdata.

the class DatasetDefinitionRegistryWithDefaultModules method testIndexedTableReconfigure.

@Test
public void testIndexedTableReconfigure() throws IncompatibleUpdateException {
    DatasetDefinition indexedTableDef = registry.get(IndexedTable.class.getName());
    Assert.assertTrue(indexedTableDef instanceof Reconfigurable);
    DatasetProperties props = TableProperties.builder().setReadlessIncrementSupport(false).add(IndexedTable.INDEX_COLUMNS_CONF_KEY, "a,b,c").build();
    DatasetSpecification spec = indexedTableDef.configure("idxtb", props);
    DatasetProperties compat = TableProperties.builder().setReadlessIncrementSupport(// turning on is ok
    true).add(IndexedTable.INDEX_COLUMNS_CONF_KEY, "c,b,a").build();
    spec = ((Reconfigurable) indexedTableDef).reconfigure("idxtb", compat, spec);
    DatasetProperties incompat = TableProperties.builder().setReadlessIncrementSupport(true).add(IndexedTable.INDEX_COLUMNS_CONF_KEY, "a,d").build();
    try {
        ((Reconfigurable) indexedTableDef).reconfigure("idxtb", incompat, spec);
        Assert.fail("reconfigure should have thrown exception");
    } catch (IncompatibleUpdateException e) {
    // expected
    }
    incompat = TableProperties.builder().setReadlessIncrementSupport(// turning off is not ok
    false).add(IndexedTable.INDEX_COLUMNS_CONF_KEY, "a,b,c").build();
    try {
        ((Reconfigurable) indexedTableDef).reconfigure("idxtb", incompat, spec);
        Assert.fail("reconfigure should have thrown exception");
    } catch (IncompatibleUpdateException e) {
    // expected
    }
}