Examples with DatasetTypeId io.cdap.cdap.proto.id.DatasetTypeId used on opensource projects

Example 6 with DatasetTypeId

use of io.cdap.cdap.proto.id.DatasetTypeId in project cdap by caskdata.

the class AuthorizableTest method testDatasetType.

@Test
public void testDatasetType() {
    DatasetTypeId datasetTypeId = new DatasetTypeId("ns", "io.cdap.test_datasetType");
    Authorizable authorizable = Authorizable.fromEntityId(datasetTypeId);
    Assert.assertEquals(datasetTypeId.toString(), authorizable.toString());
    String widcardId = datasetTypeId.toString().replace("est", "*es?t");
    Assert.assertEquals(widcardId, Authorizable.fromString(widcardId).toString());
}

Example 7 with DatasetTypeId

use of io.cdap.cdap.proto.id.DatasetTypeId in project cdap by caskdata.

the class AuthorizationTest method deployDummyAppWithImpersonation.

private void deployDummyAppWithImpersonation(NamespaceMeta nsMeta, @Nullable String appOwner) throws Exception {
    NamespaceId namespaceId = nsMeta.getNamespaceId();
    ApplicationId dummyAppId = namespaceId.app(DummyApp.class.getSimpleName());
    ArtifactId artifactId = namespaceId.artifact(DummyApp.class.getSimpleName(), "1.0-SNAPSHOT");
    DatasetId datasetId = namespaceId.dataset("whom");
    DatasetTypeId datasetTypeId = namespaceId.datasetType(KeyValueTable.class.getName());
    String owner = appOwner != null ? appOwner : nsMeta.getConfig().getPrincipal();
    KerberosPrincipalId principalId = new KerberosPrincipalId(owner);
    Principal principal = new Principal(owner, Principal.PrincipalType.USER);
    DatasetId dummyDatasetId = namespaceId.dataset("customDataset");
    DatasetTypeId dummyTypeId = namespaceId.datasetType(DummyApp.CustomDummyDataset.class.getName());
    DatasetModuleId dummyModuleId = namespaceId.datasetModule((DummyApp.CustomDummyDataset.class.getName()));
    // these are the privileges that are needed to deploy the app if no impersonation is involved,
    // can check testApps() for more info
    Map<EntityId, Set<? extends Permission>> neededPrivileges = ImmutableMap.<EntityId, Set<? extends Permission>>builder().put(dummyAppId, EnumSet.of(StandardPermission.GET, StandardPermission.CREATE)).put(artifactId, EnumSet.of(StandardPermission.CREATE)).put(datasetId, EnumSet.of(StandardPermission.CREATE, StandardPermission.GET)).put(datasetTypeId, EnumSet.of(StandardPermission.UPDATE)).put(principalId, EnumSet.of(AccessPermission.SET_OWNER)).put(dummyDatasetId, EnumSet.of(StandardPermission.CREATE, StandardPermission.GET)).put(dummyTypeId, EnumSet.of(StandardPermission.UPDATE)).put(dummyModuleId, EnumSet.of(StandardPermission.UPDATE)).build();
    setUpPrivilegeAndRegisterForDeletion(ALICE, neededPrivileges);
    // add the artifact
    addAppArtifact(artifactId, DummyApp.class);
    AppRequest<? extends Config> appRequest = new AppRequest<>(new ArtifactSummary(artifactId.getArtifact(), artifactId.getVersion()), null, appOwner);
    try {
        deployApplication(dummyAppId, appRequest);
        Assert.fail();
    } catch (Exception e) {
    // expected
    }
    // revoke privileges on datasets from alice, she does not need these privileges to deploy the app
    // the owner will need these privileges to deploy
    revokeAndAssertSuccess(datasetId);
    revokeAndAssertSuccess(datasetTypeId);
    revokeAndAssertSuccess(dummyDatasetId);
    revokeAndAssertSuccess(dummyTypeId);
    revokeAndAssertSuccess(dummyModuleId);
    // grant privileges to owner
    grantAndAssertSuccess(namespaceId, principal, EnumSet.of(StandardPermission.GET));
    grantAndAssertSuccess(datasetId, principal, EnumSet.of(StandardPermission.CREATE, StandardPermission.GET));
    grantAndAssertSuccess(datasetTypeId, principal, EnumSet.of(StandardPermission.CREATE, StandardPermission.GET));
    grantAndAssertSuccess(dummyDatasetId, principal, EnumSet.of(StandardPermission.CREATE, StandardPermission.GET));
    grantAndAssertSuccess(dummyTypeId, principal, EnumSet.of(StandardPermission.CREATE, StandardPermission.GET));
    grantAndAssertSuccess(dummyModuleId, principal, EnumSet.of(StandardPermission.CREATE, StandardPermission.GET));
    // this time it should be successful
    deployApplication(dummyAppId, appRequest);
    // clean up the privilege on the owner principal id
    revokeAndAssertSuccess(principalId);
}

Example 8 with DatasetTypeId

use of io.cdap.cdap.proto.id.DatasetTypeId in project cdap by caskdata.

the class DatasetInstanceService method getFromMds.

/**
 * Read the dataset meta data (instance and type) from MDS.
 *
 * Note this method cannot be called to create dataset instance, since it does not have enforcement on the dataset
 * type.
 */
private DatasetMeta getFromMds(DatasetId instance) throws Exception {
    // TODO: CDAP-3901 add back namespace existence check
    LOG.trace("Retrieving instance metadata from MDS for dataset {}", instance.getDataset());
    DatasetSpecification spec = instanceManager.get(instance);
    if (spec == null) {
        throw new NotFoundException(instance);
    }
    LOG.trace("Retrieved instance metadata from MDS for dataset {}", instance.getDataset());
    spec = DatasetsUtil.fixOriginalProperties(spec);
    DatasetTypeId datasetTypeId = instance.getParent().datasetType(spec.getType());
    // by pass the auth check for dataset type when the operation is not creation
    DatasetTypeMeta typeMeta = getTypeInfo(instance.getParent(), spec.getType(), true);
    if (typeMeta == null) {
        // TODO: This shouldn't happen unless CDAP is in an invalid state - maybe give different error
        throw new NotFoundException(datasetTypeId);
    }
    // for system dataset do not look up owner information in store as we know that it will be null.
    // Also, this is required for CDAP to start, because initially we don't want to look up owner admin
    // (causing its own lookup) as the SystemDatasetInitiator.getDataset is called when CDAP starts
    String ownerPrincipal = null;
    if (!NamespaceId.SYSTEM.equals(instance.getNamespaceId())) {
        LOG.trace("Retrieving owner principal for dataset {}", instance.getDataset());
        ownerPrincipal = ownerAdmin.getOwnerPrincipal(instance);
        LOG.trace("Retrieved owner principal for dataset {}", instance.getDataset());
    }
    return new DatasetMeta(spec, typeMeta, null, ownerPrincipal);
}

Example 9 with DatasetTypeId

use of io.cdap.cdap.proto.id.DatasetTypeId in project cdap by caskdata.

the class DatasetClientTestRun method testAll.

@Test
public void testAll() throws Exception {
    DatasetModuleId module = TEST_NAMESPACE.datasetModule(StandaloneDatasetModule.NAME);
    DatasetTypeId type = TEST_NAMESPACE.datasetType(StandaloneDataset.class.getName());
    DatasetModuleId moduleInOtherNamespace = OTHER_NAMESPACE.datasetModule(StandaloneDatasetModule.NAME);
    DatasetTypeId typeInOtherNamespace = OTHER_NAMESPACE.datasetType(StandaloneDataset.class.getName());
    int numBaseModules = moduleClient.list(TEST_NAMESPACE).size();
    int numBaseTypes = typeClient.list(TEST_NAMESPACE).size();
    LOG.info("Adding Dataset module");
    File moduleJarFile = createAppJarFile(StandaloneDatasetModule.class);
    moduleClient.add(TEST_NAMESPACE.datasetModule(StandaloneDatasetModule.NAME), StandaloneDatasetModule.class.getName(), moduleJarFile);
    Assert.assertEquals(numBaseModules + 1, moduleClient.list(TEST_NAMESPACE).size());
    Assert.assertEquals(numBaseTypes + 2, typeClient.list(TEST_NAMESPACE).size());
    LOG.info("Checking that the new Dataset module exists");
    DatasetModuleMeta datasetModuleMeta = moduleClient.get(module);
    Assert.assertNotNull(datasetModuleMeta);
    Assert.assertEquals(StandaloneDatasetModule.NAME, datasetModuleMeta.getName());
    LOG.info("Checking that the new Dataset module does not exist in a different namespace");
    try {
        moduleClient.get(moduleInOtherNamespace);
        Assert.fail("datasetModule found in namespace other than one in which it was expected");
    } catch (DatasetModuleNotFoundException expected) {
    // expected
    }
    LOG.info("Checking that the new Dataset type exists");
    DatasetTypeMeta datasetTypeMeta = typeClient.get(type);
    Assert.assertNotNull(datasetTypeMeta);
    Assert.assertEquals(type.getType(), datasetTypeMeta.getName());
    datasetTypeMeta = typeClient.get(type);
    Assert.assertNotNull(datasetTypeMeta);
    Assert.assertEquals(StandaloneDataset.class.getName(), datasetTypeMeta.getName());
    LOG.info("Checking that the new Dataset type does not exist in a different namespace");
    try {
        typeClient.get(typeInOtherNamespace);
        Assert.fail("datasetType found in namespace other than one in which it was expected");
    } catch (DatasetTypeNotFoundException expected) {
    // expected
    }
    LOG.info("Creating, truncating, and deleting dataset of new Dataset type");
    // Before creating dataset, there are some system datasets already exist
    int numBaseDataset = datasetClient.list(TEST_NAMESPACE).size();
    DatasetId instance = TEST_NAMESPACE.dataset("testDataset");
    String description = "test description";
    datasetClient.create(instance, new DatasetInstanceConfiguration(StandaloneDataset.TYPE_NAME, Collections.<String, String>emptyMap(), description, null));
    Assert.assertEquals(numBaseDataset + 1, datasetClient.list(TEST_NAMESPACE).size());
    // Assert dataset summary for the newly created dataset
    DatasetSpecificationSummary expectedSpec = new DatasetSpecificationSummary(instance.getDataset(), StandaloneDataset.TYPE_NAME, description, Collections.<String, String>emptyMap());
    Assert.assertEquals(expectedSpec, getSpecForDataset(instance, datasetClient.list(TEST_NAMESPACE)));
    datasetClient.truncate(instance);
    DatasetMeta metaBefore = datasetClient.get(instance);
    Assert.assertEquals(0, metaBefore.getSpec().getProperties().size());
    datasetClient.update(instance, ImmutableMap.of("sdf", "foo", "abc", "123"));
    DatasetMeta metaAfter = datasetClient.get(instance);
    Assert.assertEquals(2, metaAfter.getSpec().getProperties().size());
    Assert.assertTrue(metaAfter.getSpec().getProperties().containsKey("sdf"));
    Assert.assertTrue(metaAfter.getSpec().getProperties().containsKey("abc"));
    Assert.assertEquals("foo", metaAfter.getSpec().getProperties().get("sdf"));
    Assert.assertEquals("123", metaAfter.getSpec().getProperties().get("abc"));
    datasetClient.updateExisting(instance, ImmutableMap.of("sdf", "fzz"));
    metaAfter = datasetClient.get(instance);
    Assert.assertEquals(2, metaAfter.getSpec().getProperties().size());
    Assert.assertTrue(metaAfter.getSpec().getProperties().containsKey("sdf"));
    Assert.assertTrue(metaAfter.getSpec().getProperties().containsKey("abc"));
    Assert.assertEquals("fzz", metaAfter.getSpec().getProperties().get("sdf"));
    Assert.assertEquals("123", metaAfter.getSpec().getProperties().get("abc"));
    datasetClient.delete(instance);
    Assert.assertEquals(numBaseDataset, datasetClient.list(TEST_NAMESPACE).size());
    LOG.info("Creating and deleting multiple Datasets");
    for (int i = 1; i <= 3; i++) {
        datasetClient.create(TEST_NAMESPACE.dataset("testDataset" + i), StandaloneDataset.TYPE_NAME);
    }
    Assert.assertEquals(numBaseDataset + 3, datasetClient.list(TEST_NAMESPACE).size());
    for (int i = 1; i <= 3; i++) {
        datasetClient.delete(TEST_NAMESPACE.dataset("testDataset" + i));
    }
    Assert.assertEquals(numBaseDataset, datasetClient.list(TEST_NAMESPACE).size());
    LOG.info("Deleting Dataset module");
    moduleClient.delete(module);
    Assert.assertEquals(numBaseModules, moduleClient.list(TEST_NAMESPACE).size());
    Assert.assertEquals(numBaseTypes, typeClient.list(TEST_NAMESPACE).size());
    LOG.info("Adding Dataset module and then deleting all Dataset modules");
    moduleClient.add(TEST_NAMESPACE.datasetModule("testModule1"), StandaloneDatasetModule.class.getName(), moduleJarFile);
    Assert.assertEquals(numBaseModules + 1, moduleClient.list(TEST_NAMESPACE).size());
    Assert.assertEquals(numBaseTypes + 2, typeClient.list(TEST_NAMESPACE).size());
    moduleClient.deleteAll(TEST_NAMESPACE);
    Assert.assertEquals(numBaseModules, moduleClient.list(TEST_NAMESPACE).size());
    Assert.assertEquals(numBaseTypes, typeClient.list(TEST_NAMESPACE).size());
}

Example 10 with DatasetTypeId

use of io.cdap.cdap.proto.id.DatasetTypeId in project cdap by caskdata.

the class DatasetClientTestRun method testSystemTypes.

@Test
public void testSystemTypes() throws Exception {
    // Tests that a dataset can be created in a namespace, even if the type does not exist in that namespace.
    // The dataset type is being resolved from the system namespace.
    DatasetTypeId type = TEST_NAMESPACE.datasetType(Table.class.getName());
    DatasetId instance = TEST_NAMESPACE.dataset("tableTypeDataset");
    Assert.assertFalse(typeClient.exists(type));
    Assert.assertFalse(datasetClient.exists(instance));
    datasetClient.create(instance, Table.class.getName());
    Assert.assertTrue(datasetClient.exists(instance));
}