Examples with DatasetTypeMeta io.cdap.cdap.proto.DatasetTypeMeta used on opensource projects

Example 16 with DatasetTypeMeta

use of io.cdap.cdap.proto.DatasetTypeMeta in project cdap by caskdata.

the class DatasetAdminOpHTTPHandler method create.

@POST
@Path("/data/datasets/{name}/admin/create")
public void create(FullHttpRequest request, HttpResponder responder, @PathParam("namespace-id") String namespaceId, @PathParam("name") String name) throws Exception {
    propagateUserId(request);
    InternalDatasetCreationParams params = GSON.fromJson(request.content().toString(StandardCharsets.UTF_8), InternalDatasetCreationParams.class);
    Preconditions.checkArgument(params.getProperties() != null, "Missing required 'instanceProps' parameter.");
    Preconditions.checkArgument(params.getTypeMeta() != null, "Missing required 'typeMeta' parameter.");
    DatasetProperties props = params.getProperties();
    DatasetTypeMeta typeMeta = params.getTypeMeta();
    try {
        DatasetId instanceId = new DatasetId(namespaceId, name);
        DatasetCreationResponse response = datasetAdminService.createOrUpdate(instanceId, typeMeta, props, null);
        responder.sendJson(HttpResponseStatus.OK, GSON.toJson(response));
    } catch (IllegalArgumentException e) {
        throw new BadRequestException(e.getMessage());
    }
}

Example 17 with DatasetTypeMeta

use of io.cdap.cdap.proto.DatasetTypeMeta in project cdap by caskdata.

the class DatasetClientTestRun method testAll.

@Test
public void testAll() throws Exception {
    DatasetModuleId module = TEST_NAMESPACE.datasetModule(StandaloneDatasetModule.NAME);
    DatasetTypeId type = TEST_NAMESPACE.datasetType(StandaloneDataset.class.getName());
    DatasetModuleId moduleInOtherNamespace = OTHER_NAMESPACE.datasetModule(StandaloneDatasetModule.NAME);
    DatasetTypeId typeInOtherNamespace = OTHER_NAMESPACE.datasetType(StandaloneDataset.class.getName());
    int numBaseModules = moduleClient.list(TEST_NAMESPACE).size();
    int numBaseTypes = typeClient.list(TEST_NAMESPACE).size();
    LOG.info("Adding Dataset module");
    File moduleJarFile = createAppJarFile(StandaloneDatasetModule.class);
    moduleClient.add(TEST_NAMESPACE.datasetModule(StandaloneDatasetModule.NAME), StandaloneDatasetModule.class.getName(), moduleJarFile);
    Assert.assertEquals(numBaseModules + 1, moduleClient.list(TEST_NAMESPACE).size());
    Assert.assertEquals(numBaseTypes + 2, typeClient.list(TEST_NAMESPACE).size());
    LOG.info("Checking that the new Dataset module exists");
    DatasetModuleMeta datasetModuleMeta = moduleClient.get(module);
    Assert.assertNotNull(datasetModuleMeta);
    Assert.assertEquals(StandaloneDatasetModule.NAME, datasetModuleMeta.getName());
    LOG.info("Checking that the new Dataset module does not exist in a different namespace");
    try {
        moduleClient.get(moduleInOtherNamespace);
        Assert.fail("datasetModule found in namespace other than one in which it was expected");
    } catch (DatasetModuleNotFoundException expected) {
    // expected
    }
    LOG.info("Checking that the new Dataset type exists");
    DatasetTypeMeta datasetTypeMeta = typeClient.get(type);
    Assert.assertNotNull(datasetTypeMeta);
    Assert.assertEquals(type.getType(), datasetTypeMeta.getName());
    datasetTypeMeta = typeClient.get(type);
    Assert.assertNotNull(datasetTypeMeta);
    Assert.assertEquals(StandaloneDataset.class.getName(), datasetTypeMeta.getName());
    LOG.info("Checking that the new Dataset type does not exist in a different namespace");
    try {
        typeClient.get(typeInOtherNamespace);
        Assert.fail("datasetType found in namespace other than one in which it was expected");
    } catch (DatasetTypeNotFoundException expected) {
    // expected
    }
    LOG.info("Creating, truncating, and deleting dataset of new Dataset type");
    // Before creating dataset, there are some system datasets already exist
    int numBaseDataset = datasetClient.list(TEST_NAMESPACE).size();
    DatasetId instance = TEST_NAMESPACE.dataset("testDataset");
    String description = "test description";
    datasetClient.create(instance, new DatasetInstanceConfiguration(StandaloneDataset.TYPE_NAME, Collections.<String, String>emptyMap(), description, null));
    Assert.assertEquals(numBaseDataset + 1, datasetClient.list(TEST_NAMESPACE).size());
    // Assert dataset summary for the newly created dataset
    DatasetSpecificationSummary expectedSpec = new DatasetSpecificationSummary(instance.getDataset(), StandaloneDataset.TYPE_NAME, description, Collections.<String, String>emptyMap());
    Assert.assertEquals(expectedSpec, getSpecForDataset(instance, datasetClient.list(TEST_NAMESPACE)));
    datasetClient.truncate(instance);
    DatasetMeta metaBefore = datasetClient.get(instance);
    Assert.assertEquals(0, metaBefore.getSpec().getProperties().size());
    datasetClient.update(instance, ImmutableMap.of("sdf", "foo", "abc", "123"));
    DatasetMeta metaAfter = datasetClient.get(instance);
    Assert.assertEquals(2, metaAfter.getSpec().getProperties().size());
    Assert.assertTrue(metaAfter.getSpec().getProperties().containsKey("sdf"));
    Assert.assertTrue(metaAfter.getSpec().getProperties().containsKey("abc"));
    Assert.assertEquals("foo", metaAfter.getSpec().getProperties().get("sdf"));
    Assert.assertEquals("123", metaAfter.getSpec().getProperties().get("abc"));
    datasetClient.updateExisting(instance, ImmutableMap.of("sdf", "fzz"));
    metaAfter = datasetClient.get(instance);
    Assert.assertEquals(2, metaAfter.getSpec().getProperties().size());
    Assert.assertTrue(metaAfter.getSpec().getProperties().containsKey("sdf"));
    Assert.assertTrue(metaAfter.getSpec().getProperties().containsKey("abc"));
    Assert.assertEquals("fzz", metaAfter.getSpec().getProperties().get("sdf"));
    Assert.assertEquals("123", metaAfter.getSpec().getProperties().get("abc"));
    datasetClient.delete(instance);
    Assert.assertEquals(numBaseDataset, datasetClient.list(TEST_NAMESPACE).size());
    LOG.info("Creating and deleting multiple Datasets");
    for (int i = 1; i <= 3; i++) {
        datasetClient.create(TEST_NAMESPACE.dataset("testDataset" + i), StandaloneDataset.TYPE_NAME);
    }
    Assert.assertEquals(numBaseDataset + 3, datasetClient.list(TEST_NAMESPACE).size());
    for (int i = 1; i <= 3; i++) {
        datasetClient.delete(TEST_NAMESPACE.dataset("testDataset" + i));
    }
    Assert.assertEquals(numBaseDataset, datasetClient.list(TEST_NAMESPACE).size());
    LOG.info("Deleting Dataset module");
    moduleClient.delete(module);
    Assert.assertEquals(numBaseModules, moduleClient.list(TEST_NAMESPACE).size());
    Assert.assertEquals(numBaseTypes, typeClient.list(TEST_NAMESPACE).size());
    LOG.info("Adding Dataset module and then deleting all Dataset modules");
    moduleClient.add(TEST_NAMESPACE.datasetModule("testModule1"), StandaloneDatasetModule.class.getName(), moduleJarFile);
    Assert.assertEquals(numBaseModules + 1, moduleClient.list(TEST_NAMESPACE).size());
    Assert.assertEquals(numBaseTypes + 2, typeClient.list(TEST_NAMESPACE).size());
    moduleClient.deleteAll(TEST_NAMESPACE);
    Assert.assertEquals(numBaseModules, moduleClient.list(TEST_NAMESPACE).size());
    Assert.assertEquals(numBaseTypes, typeClient.list(TEST_NAMESPACE).size());
}

Example 18 with DatasetTypeMeta

use of io.cdap.cdap.proto.DatasetTypeMeta in project cdap by caskdata.

the class DatasetTypeClient method list.

/**
 * Lists all dataset types.
 *
 * @return list of {@link DatasetTypeMeta}s.
 * @throws IOException if a network error occurred
 * @throws UnauthenticatedException if the request is not authorized successfully in the gateway server
 */
public List<DatasetTypeMeta> list(NamespaceId namespace) throws IOException, UnauthenticatedException, UnauthorizedException {
    URL url = config.resolveNamespacedURLV3(namespace, "data/types");
    HttpResponse response = restClient.execute(HttpMethod.GET, url, config.getAccessToken());
    return ObjectResponse.fromJsonBody(response, new TypeToken<List<DatasetTypeMeta>>() {
    }).getResponseObject();
}

Example 19 with DatasetTypeMeta

use of io.cdap.cdap.proto.DatasetTypeMeta in project cdap by caskdata.

the class InMemoryDatasetFramework method addModule.

@Override
public void addModule(DatasetModuleId moduleId, DatasetModule module) throws ModuleConflictException {
    // TODO (CDAP-6297): check if existing modules overlap, or if this removes a type other modules depend on
    writeLock.lock();
    try {
        DatasetDefinitionRegistry registry = registries.get(moduleId.getParent());
        if (registry == null) {
            registry = registryFactory.create();
            registries.put(moduleId.getParent(), registry);
        }
        TypesTrackingRegistry trackingRegistry = new TypesTrackingRegistry(registry);
        module.register(trackingRegistry);
        String moduleClassName = DatasetModules.getDatasetModuleClass(module).getName();
        moduleClasses.put(moduleId.getParent(), moduleId, moduleClassName);
        List<String> types = trackingRegistry.getTypes();
        nonDefaultTypes.putAll(moduleId.getParent(), types);
        for (String type : types) {
            this.types.put(moduleId.getParent().datasetType(type), new DatasetTypeMeta(type, Collections.singletonList(new DatasetModuleMeta(moduleId.getEntityName(), moduleClassName, null, types, Collections.<String>emptyList()))));
        }
    } finally {
        writeLock.unlock();
    }
}

Example 20 with DatasetTypeMeta

use of io.cdap.cdap.proto.DatasetTypeMeta in project cdap by caskdata.

the class CLITestBase method testDataset.

@Test
public void testDataset() throws Exception {
    String datasetName = PREFIX + "sdf123lkj";
    String ownedDatasetName = PREFIX + "owned";
    CLIConfig cliConfig = getCliConfig();
    DatasetTypeClient datasetTypeClient = new DatasetTypeClient(cliConfig.getClientConfig());
    DatasetTypeMeta datasetType = datasetTypeClient.list(NamespaceId.DEFAULT).get(0);
    testCommandOutputContains("create dataset instance " + datasetType.getName() + " " + datasetName + " \"a=1\"", "Successfully created dataset");
    testCommandOutputContains("list dataset instances", FakeDataset.class.getSimpleName());
    testCommandOutputContains("get dataset instance properties " + datasetName, "a,1");
    // test dataset creation with owner
    String commandOutput = getCommandOutput("create dataset instance " + datasetType.getName() + " " + ownedDatasetName + " \"a=1\"" + " " + "someDescription " + ArgumentName.PRINCIPAL + " alice/somehost.net@somekdc.net");
    Assert.assertTrue(commandOutput.contains("Successfully created dataset"));
    Assert.assertTrue(commandOutput.contains("alice/somehost.net@somekdc.net"));
    // test describing the table returns the given owner information
    testCommandOutputContains("describe dataset instance " + ownedDatasetName, "alice/somehost.net@somekdc.net");
    NamespaceClient namespaceClient = new NamespaceClient(cliConfig.getClientConfig());
    NamespaceId barspace = new NamespaceId("bar");
    namespaceClient.create(new NamespaceMeta.Builder().setName(barspace).build());
    cliConfig.setNamespace(barspace);
    // list of dataset instances is different in 'foo' namespace
    testCommandOutputNotContains("list dataset instances", FakeDataset.class.getSimpleName());
    // also can not create dataset instances if the type it depends on exists only in a different namespace.
    DatasetTypeId datasetType1 = barspace.datasetType(datasetType.getName());
    testCommandOutputContains("create dataset instance " + datasetType.getName() + " " + datasetName, new DatasetTypeNotFoundException(datasetType1).getMessage());
    testCommandOutputContains("use namespace default", "Now using namespace 'default'");
    try {
        testCommandOutputContains("truncate dataset instance " + datasetName, "Successfully truncated");
    } finally {
        testCommandOutputContains("delete dataset instance " + datasetName, "Successfully deleted");
    }
    String datasetName2 = PREFIX + "asoijm39485";
    String description = "test-description-for-" + datasetName2;
    testCommandOutputContains("create dataset instance " + datasetType.getName() + " " + datasetName2 + " \"a=1\"" + " " + description, "Successfully created dataset");
    testCommandOutputContains("list dataset instances", description);
    testCommandOutputContains("delete dataset instance " + datasetName2, "Successfully deleted");
    testCommandOutputContains("delete dataset instance " + ownedDatasetName, "Successfully deleted");
}