use of io.cdap.cdap.proto.DatasetModuleMeta in project cdap by caskdata.
the class ListDatasetModulesCommand method perform.
@Override
public void perform(Arguments arguments, PrintStream output) throws Exception {
List<DatasetModuleMeta> modules = client.list(cliConfig.getCurrentNamespace());
Table table = Table.builder().setHeader("name", "className").setRows(modules, new RowMaker<DatasetModuleMeta>() {
@Override
public List<?> makeRow(DatasetModuleMeta object) {
return Lists.newArrayList(object.getName(), object.getClassName());
}
}).build();
cliConfig.getTableRenderer().render(cliConfig, output, table);
}
use of io.cdap.cdap.proto.DatasetModuleMeta in project cdap by caskdata.
the class DatasetClientTestRun method testAll.
@Test
public void testAll() throws Exception {
DatasetModuleId module = TEST_NAMESPACE.datasetModule(StandaloneDatasetModule.NAME);
DatasetTypeId type = TEST_NAMESPACE.datasetType(StandaloneDataset.class.getName());
DatasetModuleId moduleInOtherNamespace = OTHER_NAMESPACE.datasetModule(StandaloneDatasetModule.NAME);
DatasetTypeId typeInOtherNamespace = OTHER_NAMESPACE.datasetType(StandaloneDataset.class.getName());
int numBaseModules = moduleClient.list(TEST_NAMESPACE).size();
int numBaseTypes = typeClient.list(TEST_NAMESPACE).size();
LOG.info("Adding Dataset module");
File moduleJarFile = createAppJarFile(StandaloneDatasetModule.class);
moduleClient.add(TEST_NAMESPACE.datasetModule(StandaloneDatasetModule.NAME), StandaloneDatasetModule.class.getName(), moduleJarFile);
Assert.assertEquals(numBaseModules + 1, moduleClient.list(TEST_NAMESPACE).size());
Assert.assertEquals(numBaseTypes + 2, typeClient.list(TEST_NAMESPACE).size());
LOG.info("Checking that the new Dataset module exists");
DatasetModuleMeta datasetModuleMeta = moduleClient.get(module);
Assert.assertNotNull(datasetModuleMeta);
Assert.assertEquals(StandaloneDatasetModule.NAME, datasetModuleMeta.getName());
LOG.info("Checking that the new Dataset module does not exist in a different namespace");
try {
moduleClient.get(moduleInOtherNamespace);
Assert.fail("datasetModule found in namespace other than one in which it was expected");
} catch (DatasetModuleNotFoundException expected) {
// expected
}
LOG.info("Checking that the new Dataset type exists");
DatasetTypeMeta datasetTypeMeta = typeClient.get(type);
Assert.assertNotNull(datasetTypeMeta);
Assert.assertEquals(type.getType(), datasetTypeMeta.getName());
datasetTypeMeta = typeClient.get(type);
Assert.assertNotNull(datasetTypeMeta);
Assert.assertEquals(StandaloneDataset.class.getName(), datasetTypeMeta.getName());
LOG.info("Checking that the new Dataset type does not exist in a different namespace");
try {
typeClient.get(typeInOtherNamespace);
Assert.fail("datasetType found in namespace other than one in which it was expected");
} catch (DatasetTypeNotFoundException expected) {
// expected
}
LOG.info("Creating, truncating, and deleting dataset of new Dataset type");
// Before creating dataset, there are some system datasets already exist
int numBaseDataset = datasetClient.list(TEST_NAMESPACE).size();
DatasetId instance = TEST_NAMESPACE.dataset("testDataset");
String description = "test description";
datasetClient.create(instance, new DatasetInstanceConfiguration(StandaloneDataset.TYPE_NAME, Collections.<String, String>emptyMap(), description, null));
Assert.assertEquals(numBaseDataset + 1, datasetClient.list(TEST_NAMESPACE).size());
// Assert dataset summary for the newly created dataset
DatasetSpecificationSummary expectedSpec = new DatasetSpecificationSummary(instance.getDataset(), StandaloneDataset.TYPE_NAME, description, Collections.<String, String>emptyMap());
Assert.assertEquals(expectedSpec, getSpecForDataset(instance, datasetClient.list(TEST_NAMESPACE)));
datasetClient.truncate(instance);
DatasetMeta metaBefore = datasetClient.get(instance);
Assert.assertEquals(0, metaBefore.getSpec().getProperties().size());
datasetClient.update(instance, ImmutableMap.of("sdf", "foo", "abc", "123"));
DatasetMeta metaAfter = datasetClient.get(instance);
Assert.assertEquals(2, metaAfter.getSpec().getProperties().size());
Assert.assertTrue(metaAfter.getSpec().getProperties().containsKey("sdf"));
Assert.assertTrue(metaAfter.getSpec().getProperties().containsKey("abc"));
Assert.assertEquals("foo", metaAfter.getSpec().getProperties().get("sdf"));
Assert.assertEquals("123", metaAfter.getSpec().getProperties().get("abc"));
datasetClient.updateExisting(instance, ImmutableMap.of("sdf", "fzz"));
metaAfter = datasetClient.get(instance);
Assert.assertEquals(2, metaAfter.getSpec().getProperties().size());
Assert.assertTrue(metaAfter.getSpec().getProperties().containsKey("sdf"));
Assert.assertTrue(metaAfter.getSpec().getProperties().containsKey("abc"));
Assert.assertEquals("fzz", metaAfter.getSpec().getProperties().get("sdf"));
Assert.assertEquals("123", metaAfter.getSpec().getProperties().get("abc"));
datasetClient.delete(instance);
Assert.assertEquals(numBaseDataset, datasetClient.list(TEST_NAMESPACE).size());
LOG.info("Creating and deleting multiple Datasets");
for (int i = 1; i <= 3; i++) {
datasetClient.create(TEST_NAMESPACE.dataset("testDataset" + i), StandaloneDataset.TYPE_NAME);
}
Assert.assertEquals(numBaseDataset + 3, datasetClient.list(TEST_NAMESPACE).size());
for (int i = 1; i <= 3; i++) {
datasetClient.delete(TEST_NAMESPACE.dataset("testDataset" + i));
}
Assert.assertEquals(numBaseDataset, datasetClient.list(TEST_NAMESPACE).size());
LOG.info("Deleting Dataset module");
moduleClient.delete(module);
Assert.assertEquals(numBaseModules, moduleClient.list(TEST_NAMESPACE).size());
Assert.assertEquals(numBaseTypes, typeClient.list(TEST_NAMESPACE).size());
LOG.info("Adding Dataset module and then deleting all Dataset modules");
moduleClient.add(TEST_NAMESPACE.datasetModule("testModule1"), StandaloneDatasetModule.class.getName(), moduleJarFile);
Assert.assertEquals(numBaseModules + 1, moduleClient.list(TEST_NAMESPACE).size());
Assert.assertEquals(numBaseTypes + 2, typeClient.list(TEST_NAMESPACE).size());
moduleClient.deleteAll(TEST_NAMESPACE);
Assert.assertEquals(numBaseModules, moduleClient.list(TEST_NAMESPACE).size());
Assert.assertEquals(numBaseTypes, typeClient.list(TEST_NAMESPACE).size());
}
use of io.cdap.cdap.proto.DatasetModuleMeta in project cdap by caskdata.
the class InMemoryDatasetFramework method addModule.
@Override
public void addModule(DatasetModuleId moduleId, DatasetModule module) throws ModuleConflictException {
// TODO (CDAP-6297): check if existing modules overlap, or if this removes a type other modules depend on
writeLock.lock();
try {
DatasetDefinitionRegistry registry = registries.get(moduleId.getParent());
if (registry == null) {
registry = registryFactory.create();
registries.put(moduleId.getParent(), registry);
}
TypesTrackingRegistry trackingRegistry = new TypesTrackingRegistry(registry);
module.register(trackingRegistry);
String moduleClassName = DatasetModules.getDatasetModuleClass(module).getName();
moduleClasses.put(moduleId.getParent(), moduleId, moduleClassName);
List<String> types = trackingRegistry.getTypes();
nonDefaultTypes.putAll(moduleId.getParent(), types);
for (String type : types) {
this.types.put(moduleId.getParent().datasetType(type), new DatasetTypeMeta(type, Collections.singletonList(new DatasetModuleMeta(moduleId.getEntityName(), moduleClassName, null, types, Collections.<String>emptyList()))));
}
} finally {
writeLock.unlock();
}
}
use of io.cdap.cdap.proto.DatasetModuleMeta in project cdap by caskdata.
the class DatasetInstanceHandlerTest method testBasics.
@Test
public void testBasics() throws Exception {
// nothing has been created, modules and types list is empty
List<DatasetSpecificationSummary> instances = getInstances().getResponseObject();
// nothing in the beginning
Assert.assertEquals(0, instances.size());
try {
// create dataset instance with type that is not yet known to the system should fail
DatasetProperties props = DatasetProperties.builder().add("prop1", "val1").build();
Assert.assertEquals(HttpStatus.SC_NOT_FOUND, createInstance("dataset1", "datasetType2", props).getResponseCode());
// deploy modules
deployModule("module1", TestModule1.class);
deployModule("module2", TestModule2.class);
// create dataset instance
String description = "test instance description";
HttpResponse response = createInstance("dataset1", "datasetType2", description, props);
Assert.assertEquals(HttpStatus.SC_OK, response.getResponseCode());
// verify module cannot be deleted which type is used for the dataset
int modulesBeforeDelete = getModules().getResponseObject().size();
Assert.assertEquals(HttpStatus.SC_CONFLICT, deleteModule("module2").getResponseCode());
Assert.assertEquals(HttpStatus.SC_CONFLICT, deleteModules().getResponseCode());
Assert.assertEquals(modulesBeforeDelete, getModules().getResponseObject().size());
// Verify that dataset instance can be retrieved using specified properties
Map<String, String> properties = new HashMap<>(props.getProperties());
instances = getInstancesWithProperties(NamespaceId.DEFAULT.getNamespace(), properties).getResponseObject();
Assert.assertEquals(1, instances.size());
properties.put("some_prop_not_associated_with_dataset", "somevalue");
instances = getInstancesWithProperties(NamespaceId.DEFAULT.getNamespace(), properties).getResponseObject();
Assert.assertEquals(0, instances.size());
// verify instance was created
instances = getInstances().getResponseObject();
Assert.assertEquals(1, instances.size());
// verifying spec is same as expected
DatasetSpecification dataset1Spec = createType2Spec("dataset1", "datasetType2", description, props);
Assert.assertEquals(spec2Summary(dataset1Spec), instances.get(0));
// verify created instance info can be retrieved
DatasetMeta datasetInfo = getInstanceObject("dataset1").getResponseObject();
Assert.assertEquals(dataset1Spec, datasetInfo.getSpec());
Assert.assertEquals(dataset1Spec.getType(), datasetInfo.getType().getName());
// type meta should have 2 modules that has to be loaded to create type's class
// and in the order they must be loaded
List<DatasetModuleMeta> modules = datasetInfo.getType().getModules();
Assert.assertEquals(2, modules.size());
DatasetTypeHandlerTest.verify(modules.get(0), "module1", TestModule1.class, ImmutableList.of("datasetType1"), Collections.emptyList(), ImmutableList.of("module2"));
DatasetTypeHandlerTest.verify(modules.get(1), "module2", TestModule2.class, ImmutableList.of("datasetType2"), ImmutableList.of("module1"), Collections.emptyList());
// try to retrieve non-existed instance
Assert.assertEquals(HttpStatus.SC_NOT_FOUND, getInstance("non-existing-dataset").getResponseCode());
// cannot create instance with same name again
Assert.assertEquals(HttpStatus.SC_CONFLICT, createInstance("dataset1", "datasetType2", props).getResponseCode());
Assert.assertEquals(1, getInstances().getResponseObject().size());
// cannot delete non-existing dataset instance
Assert.assertEquals(HttpStatus.SC_NOT_FOUND, deleteInstance("non-existing-dataset").getResponseCode());
Assert.assertEquals(1, getInstances().getResponseObject().size());
// verify creation of dataset instance with null properties
Assert.assertEquals(HttpStatus.SC_OK, createInstance("nullPropertiesTable", "datasetType2").getResponseCode());
// since dataset instance description is not provided, we are using the description given by the dataset type
DatasetSpecification nullPropertiesTableSpec = createType2Spec("nullPropertiesTable", "datasetType2", TestModule2.DESCRIPTION, DatasetProperties.EMPTY);
DatasetSpecificationSummary actualSummary = getSummaryForInstance("nullPropertiesTable", getInstances().getResponseObject());
Assert.assertEquals(spec2Summary(nullPropertiesTableSpec), actualSummary);
// delete dataset instance
Assert.assertEquals(HttpStatus.SC_OK, deleteInstance("dataset1").getResponseCode());
Assert.assertEquals(HttpStatus.SC_OK, deleteInstance("nullPropertiesTable").getResponseCode());
Assert.assertEquals(0, getInstances().getResponseObject().size());
// create workflow local dataset instance
DatasetProperties localDSProperties = DatasetProperties.builder().add("prop1", "val1").add(Constants.AppFabric.WORKFLOW_LOCAL_DATASET_PROPERTY, "true").build();
Assert.assertEquals(HttpStatus.SC_OK, createInstance("localDSInstance", "datasetType2", localDSProperties).getResponseCode());
// getInstances call should still return 0
Assert.assertEquals(0, getInstances().getResponseObject().size());
Assert.assertEquals(HttpStatus.SC_OK, deleteInstance("localDSInstance").getResponseCode());
// delete dataset modules
Assert.assertEquals(HttpStatus.SC_OK, deleteModule("module2").getResponseCode());
Assert.assertEquals(HttpStatus.SC_OK, deleteModule("module1").getResponseCode());
} finally {
deleteInstance("dataset1");
deleteInstance("nullPropertiesTable");
deleteInstance("localDSInstance");
deleteModule("module2");
deleteModule("module1");
}
}
use of io.cdap.cdap.proto.DatasetModuleMeta in project cdap by caskdata.
the class DescribeDatasetModuleCommand method perform.
@Override
public void perform(Arguments arguments, PrintStream output) throws Exception {
DatasetModuleId moduleId = cliConfig.getCurrentNamespace().datasetModule(arguments.get(ArgumentName.DATASET_MODULE.toString()));
DatasetModuleMeta datasetModuleMeta = datasetModuleClient.get(moduleId);
Table table = Table.builder().setHeader("name", "className", "jarLocationPath", "types", "usesModules", "usedByModules").setRows(ImmutableList.of(datasetModuleMeta), new RowMaker<DatasetModuleMeta>() {
@Override
public List<?> makeRow(DatasetModuleMeta object) {
return Lists.newArrayList(object.getName(), object.getClassName(), object.getJarLocationPath(), Joiner.on(", ").join(object.getTypes()), Joiner.on(", ").join(object.getUsesModules()), Joiner.on(", ").join(object.getUsedByModules()));
}
}).build();
cliConfig.getTableRenderer().render(cliConfig, output, table);
}
Aggregations