use of co.cask.cdap.proto.DatasetMeta in project cdap by caskdata.
the class DatasetClientTestRun method testAll.
@Test
public void testAll() throws Exception {
DatasetModuleId module = TEST_NAMESPACE.datasetModule(StandaloneDatasetModule.NAME);
DatasetTypeId type = TEST_NAMESPACE.datasetType(StandaloneDataset.class.getName());
DatasetModuleId moduleInOtherNamespace = OTHER_NAMESPACE.datasetModule(StandaloneDatasetModule.NAME);
DatasetTypeId typeInOtherNamespace = OTHER_NAMESPACE.datasetType(StandaloneDataset.class.getName());
int numBaseModules = moduleClient.list(TEST_NAMESPACE).size();
int numBaseTypes = typeClient.list(TEST_NAMESPACE).size();
LOG.info("Adding Dataset module");
File moduleJarFile = createAppJarFile(StandaloneDatasetModule.class);
moduleClient.add(TEST_NAMESPACE.datasetModule(StandaloneDatasetModule.NAME), StandaloneDatasetModule.class.getName(), moduleJarFile);
Assert.assertEquals(numBaseModules + 1, moduleClient.list(TEST_NAMESPACE).size());
Assert.assertEquals(numBaseTypes + 2, typeClient.list(TEST_NAMESPACE).size());
LOG.info("Checking that the new Dataset module exists");
DatasetModuleMeta datasetModuleMeta = moduleClient.get(module);
Assert.assertNotNull(datasetModuleMeta);
Assert.assertEquals(StandaloneDatasetModule.NAME, datasetModuleMeta.getName());
LOG.info("Checking that the new Dataset module does not exist in a different namespace");
try {
moduleClient.get(moduleInOtherNamespace);
Assert.fail("datasetModule found in namespace other than one in which it was expected");
} catch (DatasetModuleNotFoundException expected) {
// expected
}
LOG.info("Checking that the new Dataset type exists");
DatasetTypeMeta datasetTypeMeta = typeClient.get(type);
Assert.assertNotNull(datasetTypeMeta);
Assert.assertEquals(type.getType(), datasetTypeMeta.getName());
datasetTypeMeta = typeClient.get(type);
Assert.assertNotNull(datasetTypeMeta);
Assert.assertEquals(StandaloneDataset.class.getName(), datasetTypeMeta.getName());
LOG.info("Checking that the new Dataset type does not exist in a different namespace");
try {
typeClient.get(typeInOtherNamespace);
Assert.fail("datasetType found in namespace other than one in which it was expected");
} catch (DatasetTypeNotFoundException expected) {
// expected
}
LOG.info("Creating, truncating, and deleting dataset of new Dataset type");
// Before creating dataset, there are some system datasets already exist
int numBaseDataset = datasetClient.list(TEST_NAMESPACE).size();
DatasetId instance = TEST_NAMESPACE.dataset("testDataset");
String description = "test description";
datasetClient.create(instance, new DatasetInstanceConfiguration(StandaloneDataset.TYPE_NAME, Collections.<String, String>emptyMap(), description, null));
Assert.assertEquals(numBaseDataset + 1, datasetClient.list(TEST_NAMESPACE).size());
// Assert dataset summary for the newly created dataset
DatasetSpecificationSummary expectedSpec = new DatasetSpecificationSummary(instance.getDataset(), StandaloneDataset.TYPE_NAME, description, Collections.<String, String>emptyMap());
Assert.assertEquals(expectedSpec, getSpecForDataset(instance, datasetClient.list(TEST_NAMESPACE)));
datasetClient.truncate(instance);
DatasetMeta metaBefore = datasetClient.get(instance);
Assert.assertEquals(0, metaBefore.getSpec().getProperties().size());
datasetClient.update(instance, ImmutableMap.of("sdf", "foo", "abc", "123"));
DatasetMeta metaAfter = datasetClient.get(instance);
Assert.assertEquals(2, metaAfter.getSpec().getProperties().size());
Assert.assertTrue(metaAfter.getSpec().getProperties().containsKey("sdf"));
Assert.assertTrue(metaAfter.getSpec().getProperties().containsKey("abc"));
Assert.assertEquals("foo", metaAfter.getSpec().getProperties().get("sdf"));
Assert.assertEquals("123", metaAfter.getSpec().getProperties().get("abc"));
datasetClient.updateExisting(instance, ImmutableMap.of("sdf", "fzz"));
metaAfter = datasetClient.get(instance);
Assert.assertEquals(2, metaAfter.getSpec().getProperties().size());
Assert.assertTrue(metaAfter.getSpec().getProperties().containsKey("sdf"));
Assert.assertTrue(metaAfter.getSpec().getProperties().containsKey("abc"));
Assert.assertEquals("fzz", metaAfter.getSpec().getProperties().get("sdf"));
Assert.assertEquals("123", metaAfter.getSpec().getProperties().get("abc"));
datasetClient.delete(instance);
Assert.assertEquals(numBaseDataset, datasetClient.list(TEST_NAMESPACE).size());
LOG.info("Creating and deleting multiple Datasets");
for (int i = 1; i <= 3; i++) {
datasetClient.create(TEST_NAMESPACE.dataset("testDataset" + i), StandaloneDataset.TYPE_NAME);
}
Assert.assertEquals(numBaseDataset + 3, datasetClient.list(TEST_NAMESPACE).size());
for (int i = 1; i <= 3; i++) {
datasetClient.delete(TEST_NAMESPACE.dataset("testDataset" + i));
}
Assert.assertEquals(numBaseDataset, datasetClient.list(TEST_NAMESPACE).size());
LOG.info("Deleting Dataset module");
moduleClient.delete(module);
Assert.assertEquals(numBaseModules, moduleClient.list(TEST_NAMESPACE).size());
Assert.assertEquals(numBaseTypes, typeClient.list(TEST_NAMESPACE).size());
LOG.info("Adding Dataset module and then deleting all Dataset modules");
moduleClient.add(TEST_NAMESPACE.datasetModule("testModule1"), StandaloneDatasetModule.class.getName(), moduleJarFile);
Assert.assertEquals(numBaseModules + 1, moduleClient.list(TEST_NAMESPACE).size());
Assert.assertEquals(numBaseTypes + 2, typeClient.list(TEST_NAMESPACE).size());
moduleClient.deleteAll(TEST_NAMESPACE);
Assert.assertEquals(numBaseModules, moduleClient.list(TEST_NAMESPACE).size());
Assert.assertEquals(numBaseTypes, typeClient.list(TEST_NAMESPACE).size());
}
use of co.cask.cdap.proto.DatasetMeta in project cdap by caskdata.
the class DescribeDatasetInstanceCommand method perform.
@Override
public void perform(Arguments arguments, PrintStream output) throws Exception {
DatasetId instance = cliConfig.getCurrentNamespace().dataset(arguments.get(ArgumentName.DATASET.toString()));
DatasetMeta meta = datasetClient.get(instance);
Table table = Table.builder().setHeader("hive table", "spec", "type", "principal").setRows(ImmutableList.of(meta), new RowMaker<DatasetMeta>() {
@Override
public List<?> makeRow(DatasetMeta object) {
return Lists.newArrayList(object.getHiveTableName(), GSON.toJson(object.getSpec()), GSON.toJson(object.getType()), object.getOwnerPrincipal());
}
}).build();
cliConfig.getTableRenderer().render(cliConfig, output, table);
}
use of co.cask.cdap.proto.DatasetMeta in project cdap by caskdata.
the class RemoteDatasetFramework method getAdmin.
@Nullable
@Override
public <T extends DatasetAdmin> T getAdmin(DatasetId datasetInstanceId, @Nullable ClassLoader parentClassLoader, DatasetClassLoaderProvider classLoaderProvider) throws DatasetManagementException, IOException {
DatasetMeta instanceInfo = clientCache.getUnchecked(datasetInstanceId.getParent()).getInstance(datasetInstanceId.getEntityName());
if (instanceInfo == null) {
return null;
}
DatasetType type = getType(instanceInfo.getType(), parentClassLoader, classLoaderProvider);
return (T) type.getAdmin(DatasetContext.from(datasetInstanceId.getNamespace()), instanceInfo.getSpec());
}
use of co.cask.cdap.proto.DatasetMeta in project cdap by caskdata.
the class DatasetClient method get.
/**
* Gets information about a dataset.
*
* @param instance ID of the dataset instance
* @return a {@link DatasetSpecificationSummary}.
* @throws NotFoundException if the dataset is not found
* @throws IOException if a network error occurred
* @throws UnauthenticatedException if the request is not authorized successfully in the gateway server
*/
public DatasetMeta get(DatasetId instance) throws IOException, UnauthenticatedException, NotFoundException, UnauthorizedException {
URL url = config.resolveNamespacedURLV3(instance.getParent(), String.format("data/datasets/%s", instance.getDataset()));
HttpResponse response = restClient.execute(HttpMethod.GET, url, config.getAccessToken());
if (response.getResponseCode() == HttpURLConnection.HTTP_NOT_FOUND) {
throw new NotFoundException(instance);
}
return ObjectResponse.fromJsonBody(response, DatasetMeta.class).getResponseObject();
}
use of co.cask.cdap.proto.DatasetMeta in project cdap by caskdata.
the class DatasetInstanceHandlerTest method testUpdateInstance.
@Test
public void testUpdateInstance() throws Exception {
// nothing has been created, modules and types list is empty
List<DatasetSpecificationSummary> instances = getInstances().getResponseObject();
// nothing in the beginning
Assert.assertEquals(0, instances.size());
try {
DatasetProperties props = DatasetProperties.builder().add("prop1", "val1").add(TestModule2.NOT_RECONFIGURABLE, "this").build();
// deploy modules
deployModule("module1", TestModule1.class);
deployModule("module2", TestModule2.class);
// create dataset instance
Assert.assertEquals(HttpStatus.SC_OK, createInstance("dataset1", "datasetType2", props).getResponseCode());
// verify instance was created
instances = getInstances().getResponseObject();
Assert.assertEquals(1, instances.size());
DatasetMeta meta = getInstanceObject("dataset1").getResponseObject();
Map<String, String> storedOriginalProps = meta.getSpec().getOriginalProperties();
Assert.assertEquals(props.getProperties(), storedOriginalProps);
Map<String, String> retrievedProps = getInstanceProperties("dataset1").getResponseObject();
Assert.assertEquals(props.getProperties(), retrievedProps);
// these properties are incompatible because TestModule1.NOT_RECONFIGURABLE may not change
DatasetProperties newProps = DatasetProperties.builder().add("prop2", "val2").add(TestModule2.NOT_RECONFIGURABLE, "that").build();
Assert.assertEquals(HttpStatus.SC_CONFLICT, updateInstance("dataset1", newProps).getResponseCode());
// update dataset instance with valid properties
newProps = DatasetProperties.builder().add("prop2", "val2").add(TestModule2.NOT_RECONFIGURABLE, "this").build();
Assert.assertEquals(HttpStatus.SC_OK, updateInstance("dataset1", newProps).getResponseCode());
meta = getInstanceObject("dataset1").getResponseObject();
Assert.assertEquals(newProps.getProperties(), meta.getSpec().getOriginalProperties());
Assert.assertEquals("val2", meta.getSpec().getProperty("prop2"));
Assert.assertNull(meta.getSpec().getProperty("prop1"));
retrievedProps = getInstanceProperties("dataset1").getResponseObject();
Assert.assertEquals(newProps.getProperties(), retrievedProps);
} finally {
// delete dataset instance
Assert.assertEquals(HttpStatus.SC_OK, deleteInstance("dataset1").getResponseCode());
Assert.assertEquals(0, getInstances().getResponseObject().size());
// delete dataset modules
Assert.assertEquals(HttpStatus.SC_OK, deleteModule("module2").getResponseCode());
Assert.assertEquals(HttpStatus.SC_OK, deleteModule("module1").getResponseCode());
}
}
Aggregations