Search in sources :

Example 6 with DatasetMeta

use of co.cask.cdap.proto.DatasetMeta in project cdap by caskdata.

the class DatasetClientTestRun method testAll.

@Test
public void testAll() throws Exception {
    DatasetModuleId module = TEST_NAMESPACE.datasetModule(StandaloneDatasetModule.NAME);
    DatasetTypeId type = TEST_NAMESPACE.datasetType(StandaloneDataset.class.getName());
    DatasetModuleId moduleInOtherNamespace = OTHER_NAMESPACE.datasetModule(StandaloneDatasetModule.NAME);
    DatasetTypeId typeInOtherNamespace = OTHER_NAMESPACE.datasetType(StandaloneDataset.class.getName());
    int numBaseModules = moduleClient.list(TEST_NAMESPACE).size();
    int numBaseTypes = typeClient.list(TEST_NAMESPACE).size();
    LOG.info("Adding Dataset module");
    File moduleJarFile = createAppJarFile(StandaloneDatasetModule.class);
    moduleClient.add(TEST_NAMESPACE.datasetModule(StandaloneDatasetModule.NAME), StandaloneDatasetModule.class.getName(), moduleJarFile);
    Assert.assertEquals(numBaseModules + 1, moduleClient.list(TEST_NAMESPACE).size());
    Assert.assertEquals(numBaseTypes + 2, typeClient.list(TEST_NAMESPACE).size());
    LOG.info("Checking that the new Dataset module exists");
    DatasetModuleMeta datasetModuleMeta = moduleClient.get(module);
    Assert.assertNotNull(datasetModuleMeta);
    Assert.assertEquals(StandaloneDatasetModule.NAME, datasetModuleMeta.getName());
    LOG.info("Checking that the new Dataset module does not exist in a different namespace");
    try {
        moduleClient.get(moduleInOtherNamespace);
        Assert.fail("datasetModule found in namespace other than one in which it was expected");
    } catch (DatasetModuleNotFoundException expected) {
    // expected
    }
    LOG.info("Checking that the new Dataset type exists");
    DatasetTypeMeta datasetTypeMeta = typeClient.get(type);
    Assert.assertNotNull(datasetTypeMeta);
    Assert.assertEquals(type.getType(), datasetTypeMeta.getName());
    datasetTypeMeta = typeClient.get(type);
    Assert.assertNotNull(datasetTypeMeta);
    Assert.assertEquals(StandaloneDataset.class.getName(), datasetTypeMeta.getName());
    LOG.info("Checking that the new Dataset type does not exist in a different namespace");
    try {
        typeClient.get(typeInOtherNamespace);
        Assert.fail("datasetType found in namespace other than one in which it was expected");
    } catch (DatasetTypeNotFoundException expected) {
    // expected
    }
    LOG.info("Creating, truncating, and deleting dataset of new Dataset type");
    // Before creating dataset, there are some system datasets already exist
    int numBaseDataset = datasetClient.list(TEST_NAMESPACE).size();
    DatasetId instance = TEST_NAMESPACE.dataset("testDataset");
    String description = "test description";
    datasetClient.create(instance, new DatasetInstanceConfiguration(StandaloneDataset.TYPE_NAME, Collections.<String, String>emptyMap(), description, null));
    Assert.assertEquals(numBaseDataset + 1, datasetClient.list(TEST_NAMESPACE).size());
    // Assert dataset summary for the newly created dataset
    DatasetSpecificationSummary expectedSpec = new DatasetSpecificationSummary(instance.getDataset(), StandaloneDataset.TYPE_NAME, description, Collections.<String, String>emptyMap());
    Assert.assertEquals(expectedSpec, getSpecForDataset(instance, datasetClient.list(TEST_NAMESPACE)));
    datasetClient.truncate(instance);
    DatasetMeta metaBefore = datasetClient.get(instance);
    Assert.assertEquals(0, metaBefore.getSpec().getProperties().size());
    datasetClient.update(instance, ImmutableMap.of("sdf", "foo", "abc", "123"));
    DatasetMeta metaAfter = datasetClient.get(instance);
    Assert.assertEquals(2, metaAfter.getSpec().getProperties().size());
    Assert.assertTrue(metaAfter.getSpec().getProperties().containsKey("sdf"));
    Assert.assertTrue(metaAfter.getSpec().getProperties().containsKey("abc"));
    Assert.assertEquals("foo", metaAfter.getSpec().getProperties().get("sdf"));
    Assert.assertEquals("123", metaAfter.getSpec().getProperties().get("abc"));
    datasetClient.updateExisting(instance, ImmutableMap.of("sdf", "fzz"));
    metaAfter = datasetClient.get(instance);
    Assert.assertEquals(2, metaAfter.getSpec().getProperties().size());
    Assert.assertTrue(metaAfter.getSpec().getProperties().containsKey("sdf"));
    Assert.assertTrue(metaAfter.getSpec().getProperties().containsKey("abc"));
    Assert.assertEquals("fzz", metaAfter.getSpec().getProperties().get("sdf"));
    Assert.assertEquals("123", metaAfter.getSpec().getProperties().get("abc"));
    datasetClient.delete(instance);
    Assert.assertEquals(numBaseDataset, datasetClient.list(TEST_NAMESPACE).size());
    LOG.info("Creating and deleting multiple Datasets");
    for (int i = 1; i <= 3; i++) {
        datasetClient.create(TEST_NAMESPACE.dataset("testDataset" + i), StandaloneDataset.TYPE_NAME);
    }
    Assert.assertEquals(numBaseDataset + 3, datasetClient.list(TEST_NAMESPACE).size());
    for (int i = 1; i <= 3; i++) {
        datasetClient.delete(TEST_NAMESPACE.dataset("testDataset" + i));
    }
    Assert.assertEquals(numBaseDataset, datasetClient.list(TEST_NAMESPACE).size());
    LOG.info("Deleting Dataset module");
    moduleClient.delete(module);
    Assert.assertEquals(numBaseModules, moduleClient.list(TEST_NAMESPACE).size());
    Assert.assertEquals(numBaseTypes, typeClient.list(TEST_NAMESPACE).size());
    LOG.info("Adding Dataset module and then deleting all Dataset modules");
    moduleClient.add(TEST_NAMESPACE.datasetModule("testModule1"), StandaloneDatasetModule.class.getName(), moduleJarFile);
    Assert.assertEquals(numBaseModules + 1, moduleClient.list(TEST_NAMESPACE).size());
    Assert.assertEquals(numBaseTypes + 2, typeClient.list(TEST_NAMESPACE).size());
    moduleClient.deleteAll(TEST_NAMESPACE);
    Assert.assertEquals(numBaseModules, moduleClient.list(TEST_NAMESPACE).size());
    Assert.assertEquals(numBaseTypes, typeClient.list(TEST_NAMESPACE).size());
}
Also used : DatasetTypeId(co.cask.cdap.proto.id.DatasetTypeId) StandaloneDatasetModule(co.cask.cdap.client.app.StandaloneDatasetModule) DatasetTypeMeta(co.cask.cdap.proto.DatasetTypeMeta) DatasetMeta(co.cask.cdap.proto.DatasetMeta) DatasetInstanceConfiguration(co.cask.cdap.proto.DatasetInstanceConfiguration) DatasetSpecificationSummary(co.cask.cdap.proto.DatasetSpecificationSummary) DatasetId(co.cask.cdap.proto.id.DatasetId) DatasetModuleNotFoundException(co.cask.cdap.common.DatasetModuleNotFoundException) DatasetModuleId(co.cask.cdap.proto.id.DatasetModuleId) DatasetModuleMeta(co.cask.cdap.proto.DatasetModuleMeta) StandaloneDataset(co.cask.cdap.client.app.StandaloneDataset) DatasetTypeNotFoundException(co.cask.cdap.common.DatasetTypeNotFoundException) File(java.io.File) Test(org.junit.Test)

Example 7 with DatasetMeta

use of co.cask.cdap.proto.DatasetMeta in project cdap by caskdata.

the class DescribeDatasetInstanceCommand method perform.

@Override
public void perform(Arguments arguments, PrintStream output) throws Exception {
    DatasetId instance = cliConfig.getCurrentNamespace().dataset(arguments.get(ArgumentName.DATASET.toString()));
    DatasetMeta meta = datasetClient.get(instance);
    Table table = Table.builder().setHeader("hive table", "spec", "type", "principal").setRows(ImmutableList.of(meta), new RowMaker<DatasetMeta>() {

        @Override
        public List<?> makeRow(DatasetMeta object) {
            return Lists.newArrayList(object.getHiveTableName(), GSON.toJson(object.getSpec()), GSON.toJson(object.getType()), object.getOwnerPrincipal());
        }
    }).build();
    cliConfig.getTableRenderer().render(cliConfig, output, table);
}
Also used : Table(co.cask.cdap.cli.util.table.Table) RowMaker(co.cask.cdap.cli.util.RowMaker) DatasetMeta(co.cask.cdap.proto.DatasetMeta) DatasetId(co.cask.cdap.proto.id.DatasetId)

Example 8 with DatasetMeta

use of co.cask.cdap.proto.DatasetMeta in project cdap by caskdata.

the class RemoteDatasetFramework method getAdmin.

@Nullable
@Override
public <T extends DatasetAdmin> T getAdmin(DatasetId datasetInstanceId, @Nullable ClassLoader parentClassLoader, DatasetClassLoaderProvider classLoaderProvider) throws DatasetManagementException, IOException {
    DatasetMeta instanceInfo = clientCache.getUnchecked(datasetInstanceId.getParent()).getInstance(datasetInstanceId.getEntityName());
    if (instanceInfo == null) {
        return null;
    }
    DatasetType type = getType(instanceInfo.getType(), parentClassLoader, classLoaderProvider);
    return (T) type.getAdmin(DatasetContext.from(datasetInstanceId.getNamespace()), instanceInfo.getSpec());
}
Also used : DatasetMeta(co.cask.cdap.proto.DatasetMeta) Nullable(javax.annotation.Nullable)

Example 9 with DatasetMeta

use of co.cask.cdap.proto.DatasetMeta in project cdap by caskdata.

the class DatasetClient method get.

/**
   * Gets information about a dataset.
   *
   * @param instance ID of the dataset instance
   * @return a {@link DatasetSpecificationSummary}.
   * @throws NotFoundException if the dataset is not found
   * @throws IOException if a network error occurred
   * @throws UnauthenticatedException if the request is not authorized successfully in the gateway server
   */
public DatasetMeta get(DatasetId instance) throws IOException, UnauthenticatedException, NotFoundException, UnauthorizedException {
    URL url = config.resolveNamespacedURLV3(instance.getParent(), String.format("data/datasets/%s", instance.getDataset()));
    HttpResponse response = restClient.execute(HttpMethod.GET, url, config.getAccessToken());
    if (response.getResponseCode() == HttpURLConnection.HTTP_NOT_FOUND) {
        throw new NotFoundException(instance);
    }
    return ObjectResponse.fromJsonBody(response, DatasetMeta.class).getResponseObject();
}
Also used : HttpResponse(co.cask.common.http.HttpResponse) DatasetNotFoundException(co.cask.cdap.common.DatasetNotFoundException) DatasetTypeNotFoundException(co.cask.cdap.common.DatasetTypeNotFoundException) NotFoundException(co.cask.cdap.common.NotFoundException) DatasetMeta(co.cask.cdap.proto.DatasetMeta) URL(java.net.URL)

Example 10 with DatasetMeta

use of co.cask.cdap.proto.DatasetMeta in project cdap by caskdata.

the class DatasetInstanceHandlerTest method testUpdateInstance.

@Test
public void testUpdateInstance() throws Exception {
    // nothing has been created, modules and types list is empty
    List<DatasetSpecificationSummary> instances = getInstances().getResponseObject();
    // nothing in the beginning
    Assert.assertEquals(0, instances.size());
    try {
        DatasetProperties props = DatasetProperties.builder().add("prop1", "val1").add(TestModule2.NOT_RECONFIGURABLE, "this").build();
        // deploy modules
        deployModule("module1", TestModule1.class);
        deployModule("module2", TestModule2.class);
        // create dataset instance
        Assert.assertEquals(HttpStatus.SC_OK, createInstance("dataset1", "datasetType2", props).getResponseCode());
        // verify instance was created
        instances = getInstances().getResponseObject();
        Assert.assertEquals(1, instances.size());
        DatasetMeta meta = getInstanceObject("dataset1").getResponseObject();
        Map<String, String> storedOriginalProps = meta.getSpec().getOriginalProperties();
        Assert.assertEquals(props.getProperties(), storedOriginalProps);
        Map<String, String> retrievedProps = getInstanceProperties("dataset1").getResponseObject();
        Assert.assertEquals(props.getProperties(), retrievedProps);
        // these properties are incompatible because TestModule1.NOT_RECONFIGURABLE may not change
        DatasetProperties newProps = DatasetProperties.builder().add("prop2", "val2").add(TestModule2.NOT_RECONFIGURABLE, "that").build();
        Assert.assertEquals(HttpStatus.SC_CONFLICT, updateInstance("dataset1", newProps).getResponseCode());
        // update dataset instance with valid properties
        newProps = DatasetProperties.builder().add("prop2", "val2").add(TestModule2.NOT_RECONFIGURABLE, "this").build();
        Assert.assertEquals(HttpStatus.SC_OK, updateInstance("dataset1", newProps).getResponseCode());
        meta = getInstanceObject("dataset1").getResponseObject();
        Assert.assertEquals(newProps.getProperties(), meta.getSpec().getOriginalProperties());
        Assert.assertEquals("val2", meta.getSpec().getProperty("prop2"));
        Assert.assertNull(meta.getSpec().getProperty("prop1"));
        retrievedProps = getInstanceProperties("dataset1").getResponseObject();
        Assert.assertEquals(newProps.getProperties(), retrievedProps);
    } finally {
        // delete dataset instance
        Assert.assertEquals(HttpStatus.SC_OK, deleteInstance("dataset1").getResponseCode());
        Assert.assertEquals(0, getInstances().getResponseObject().size());
        // delete dataset modules
        Assert.assertEquals(HttpStatus.SC_OK, deleteModule("module2").getResponseCode());
        Assert.assertEquals(HttpStatus.SC_OK, deleteModule("module1").getResponseCode());
    }
}
Also used : DatasetProperties(co.cask.cdap.api.dataset.DatasetProperties) DatasetMeta(co.cask.cdap.proto.DatasetMeta) DatasetSpecificationSummary(co.cask.cdap.proto.DatasetSpecificationSummary) Test(org.junit.Test)

Aggregations

DatasetMeta (co.cask.cdap.proto.DatasetMeta)12 Test (org.junit.Test)5 DatasetTypeNotFoundException (co.cask.cdap.common.DatasetTypeNotFoundException)4 NotFoundException (co.cask.cdap.common.NotFoundException)4 DatasetNotFoundException (co.cask.cdap.common.DatasetNotFoundException)3 DatasetSpecificationSummary (co.cask.cdap.proto.DatasetSpecificationSummary)3 HttpResponse (co.cask.common.http.HttpResponse)3 DatasetProperties (co.cask.cdap.api.dataset.DatasetProperties)2 DatasetSpecification (co.cask.cdap.api.dataset.DatasetSpecification)2 NamespaceNotFoundException (co.cask.cdap.common.NamespaceNotFoundException)2 DatasetInstanceConfiguration (co.cask.cdap.proto.DatasetInstanceConfiguration)2 DatasetModuleMeta (co.cask.cdap.proto.DatasetModuleMeta)2 DatasetTypeMeta (co.cask.cdap.proto.DatasetTypeMeta)2 DatasetId (co.cask.cdap.proto.id.DatasetId)2 DatasetTypeId (co.cask.cdap.proto.id.DatasetTypeId)2 Nullable (javax.annotation.Nullable)2 HttpErrorStatusProvider (co.cask.cdap.api.common.HttpErrorStatusProvider)1 RowMaker (co.cask.cdap.cli.util.RowMaker)1 Table (co.cask.cdap.cli.util.table.Table)1 StandaloneDataset (co.cask.cdap.client.app.StandaloneDataset)1