Examples with DatasetModuleId io.cdap.cdap.proto.id.DatasetModuleId used on opensource projects

Example 11 with DatasetModuleId

use of io.cdap.cdap.proto.id.DatasetModuleId in project cdap by caskdata.

the class DatasetTypeManager method deleteModule.

/**
 * Deletes specified dataset module
 * @param datasetModuleId {@link DatasetModuleId} of the dataset module to delete
 * @return true if deleted successfully, false if module didn't exist: nothing to delete
 * @throws DatasetModuleConflictException when there are other modules depend on the specified one, in which case
 *         deletion does NOT happen
 */
public boolean deleteModule(final DatasetModuleId datasetModuleId) throws DatasetModuleConflictException {
    LOG.info("Deleting module {}", datasetModuleId);
    try {
        return TransactionRunners.run(transactionRunner, context -> {
            final DatasetTypeTable datasetTypeTable = DatasetTypeTable.create(context);
            final DatasetInstanceTable datasetInstanceTable = new DatasetInstanceTable(context);
            final DatasetModuleMeta module = datasetTypeTable.getModule(datasetModuleId);
            if (module == null) {
                return false;
            }
            // cannot delete when there's module that uses it
            if (module.getUsedByModules().size() > 0) {
                String msg = String.format("Cannot delete module %s: other modules depend on it. Delete them first", module);
                throw new DatasetModuleConflictException(msg);
            }
            Collection<DatasetSpecification> instances = datasetInstanceTable.getByTypes(datasetModuleId.getParent(), ImmutableSet.copyOf(module.getTypes()));
            // cannot delete when there's instance that uses it
            if (!instances.isEmpty()) {
                String msg = String.format("Cannot delete module %s: other instances depend on it. Delete them first", module);
                throw new DatasetModuleConflictException(msg);
            }
            // remove it from "usedBy" from other modules
            for (String usedModuleName : module.getUsesModules()) {
                DatasetModuleId usedModuleId = new DatasetModuleId(datasetModuleId.getNamespace(), usedModuleName);
                // not using getModuleWithFallback here because we want to know the namespace in which usedModule was found,
                // so we can overwrite it in the MDS in the appropriate namespace
                DatasetModuleMeta usedModule = datasetTypeTable.getModule(usedModuleId);
                // if the usedModule is not found in the current namespace, try finding it in the system namespace
                if (usedModule == null) {
                    usedModuleId = NamespaceId.SYSTEM.datasetModule(usedModuleName);
                    usedModule = datasetTypeTable.getModule(usedModuleId);
                    Preconditions.checkState(usedModule != null, "Could not find a module %s that the module %s uses.", usedModuleName, datasetModuleId.getEntityName());
                }
                usedModule.removeUsedByModule(datasetModuleId.getEntityName());
                datasetTypeTable.writeModule(usedModuleId.getParent(), usedModule);
            }
            datasetTypeTable.deleteModule(datasetModuleId);
            try {
                // Also delete module jar
                Location moduleJarLocation = impersonator.doAs(datasetModuleId, () -> Locations.getLocationFromAbsolutePath(locationFactory, module.getJarLocationPath()));
                if (!moduleJarLocation.delete()) {
                    LOG.debug("Could not delete dataset module archive");
                }
            } catch (Exception e) {
                // the only checked exception the try-catch throws is IOException
                Throwables.propagateIfInstanceOf(e, IOException.class);
                throw Throwables.propagate(e);
            }
            return true;
        });
    } catch (RuntimeException e) {
        for (Throwable cause : Throwables.getCausalChain(e)) {
            if (cause instanceof DatasetModuleConflictException) {
                throw (DatasetModuleConflictException) cause;
            }
        }
        throw Throwables.propagate(e);
    } catch (Exception e) {
        LOG.error("Operation failed", e);
        throw Throwables.propagate(e);
    }
}

Example 12 with DatasetModuleId

use of io.cdap.cdap.proto.id.DatasetModuleId in project cdap by caskdata.

the class GenerateClientUsageExample method datasetModuleClient.

public void datasetModuleClient() throws Exception {
    // Construct the client used to interact with CDAP
    DatasetModuleClient datasetModuleClient = new DatasetModuleClient(clientConfig);
    // Add a dataset module
    File moduleJarFile = createAppJarFile(SomeDatasetModule.class);
    DatasetModuleId datasetModuleId = NamespaceId.DEFAULT.datasetModule("someDatasetModule");
    datasetModuleClient.add(datasetModuleId, SomeDatasetModule.class.getName(), moduleJarFile);
    // Fetch the dataset module information
    DatasetModuleMeta datasetModuleMeta = datasetModuleClient.get(datasetModuleId);
    // Delete all dataset modules
    datasetModuleClient.deleteAll(NamespaceId.DEFAULT);
}

Example 13 with DatasetModuleId

use of io.cdap.cdap.proto.id.DatasetModuleId in project cdap by caskdata.

the class AuthorizableTest method testDatasetModule.

@Test
public void testDatasetModule() {
    DatasetModuleId datasetModuleId = new DatasetModuleId("ns", "io.cdap.test_datasetModule");
    Authorizable authorizable = Authorizable.fromEntityId(datasetModuleId);
    Assert.assertEquals(datasetModuleId.toString(), authorizable.toString());
    String widcardId = datasetModuleId.toString().replace("est", "*es?t");
    Assert.assertEquals(widcardId, Authorizable.fromString(widcardId).toString());
}

Example 14 with DatasetModuleId

use of io.cdap.cdap.proto.id.DatasetModuleId in project cdap by caskdata.

the class AuthorizationTest method deployDummyAppWithImpersonation.

private void deployDummyAppWithImpersonation(NamespaceMeta nsMeta, @Nullable String appOwner) throws Exception {
    NamespaceId namespaceId = nsMeta.getNamespaceId();
    ApplicationId dummyAppId = namespaceId.app(DummyApp.class.getSimpleName());
    ArtifactId artifactId = namespaceId.artifact(DummyApp.class.getSimpleName(), "1.0-SNAPSHOT");
    DatasetId datasetId = namespaceId.dataset("whom");
    DatasetTypeId datasetTypeId = namespaceId.datasetType(KeyValueTable.class.getName());
    String owner = appOwner != null ? appOwner : nsMeta.getConfig().getPrincipal();
    KerberosPrincipalId principalId = new KerberosPrincipalId(owner);
    Principal principal = new Principal(owner, Principal.PrincipalType.USER);
    DatasetId dummyDatasetId = namespaceId.dataset("customDataset");
    DatasetTypeId dummyTypeId = namespaceId.datasetType(DummyApp.CustomDummyDataset.class.getName());
    DatasetModuleId dummyModuleId = namespaceId.datasetModule((DummyApp.CustomDummyDataset.class.getName()));
    // these are the privileges that are needed to deploy the app if no impersonation is involved,
    // can check testApps() for more info
    Map<EntityId, Set<? extends Permission>> neededPrivileges = ImmutableMap.<EntityId, Set<? extends Permission>>builder().put(dummyAppId, EnumSet.of(StandardPermission.GET, StandardPermission.CREATE)).put(artifactId, EnumSet.of(StandardPermission.CREATE)).put(datasetId, EnumSet.of(StandardPermission.CREATE, StandardPermission.GET)).put(datasetTypeId, EnumSet.of(StandardPermission.UPDATE)).put(principalId, EnumSet.of(AccessPermission.SET_OWNER)).put(dummyDatasetId, EnumSet.of(StandardPermission.CREATE, StandardPermission.GET)).put(dummyTypeId, EnumSet.of(StandardPermission.UPDATE)).put(dummyModuleId, EnumSet.of(StandardPermission.UPDATE)).build();
    setUpPrivilegeAndRegisterForDeletion(ALICE, neededPrivileges);
    // add the artifact
    addAppArtifact(artifactId, DummyApp.class);
    AppRequest<? extends Config> appRequest = new AppRequest<>(new ArtifactSummary(artifactId.getArtifact(), artifactId.getVersion()), null, appOwner);
    try {
        deployApplication(dummyAppId, appRequest);
        Assert.fail();
    } catch (Exception e) {
    // expected
    }
    // revoke privileges on datasets from alice, she does not need these privileges to deploy the app
    // the owner will need these privileges to deploy
    revokeAndAssertSuccess(datasetId);
    revokeAndAssertSuccess(datasetTypeId);
    revokeAndAssertSuccess(dummyDatasetId);
    revokeAndAssertSuccess(dummyTypeId);
    revokeAndAssertSuccess(dummyModuleId);
    // grant privileges to owner
    grantAndAssertSuccess(namespaceId, principal, EnumSet.of(StandardPermission.GET));
    grantAndAssertSuccess(datasetId, principal, EnumSet.of(StandardPermission.CREATE, StandardPermission.GET));
    grantAndAssertSuccess(datasetTypeId, principal, EnumSet.of(StandardPermission.CREATE, StandardPermission.GET));
    grantAndAssertSuccess(dummyDatasetId, principal, EnumSet.of(StandardPermission.CREATE, StandardPermission.GET));
    grantAndAssertSuccess(dummyTypeId, principal, EnumSet.of(StandardPermission.CREATE, StandardPermission.GET));
    grantAndAssertSuccess(dummyModuleId, principal, EnumSet.of(StandardPermission.CREATE, StandardPermission.GET));
    // this time it should be successful
    deployApplication(dummyAppId, appRequest);
    // clean up the privilege on the owner principal id
    revokeAndAssertSuccess(principalId);
}

Example 15 with DatasetModuleId

use of io.cdap.cdap.proto.id.DatasetModuleId in project cdap by cdapio.

the class DatasetModulesDeployer method loadAndDeployModule.

private void loadAndDeployModule(ClassLoader artifactClassLoader, String className, final Location jarLocation, String moduleName, NamespaceId namespaceId, String authorizingUser) throws Exception {
    // note: using app class loader to load module class
    @SuppressWarnings("unchecked") Class<Dataset> clazz = (Class<Dataset>) artifactClassLoader.loadClass(className);
    try {
        // note: we can deploy module or create module from Dataset class
        // note: it seems dangerous to instantiate dataset module here, but this will be fine when we move deploy into
        // isolated user's environment (e.g. separate yarn container)
        final DatasetModuleId moduleId = namespaceId.datasetModule(moduleName);
        final DatasetModule module;
        if (DatasetModule.class.isAssignableFrom(clazz)) {
            module = (DatasetModule) clazz.newInstance();
        } else if (Dataset.class.isAssignableFrom(clazz)) {
            if (systemDatasetFramework.hasSystemType(clazz.getName())) {
                return;
            }
            final DatasetTypeId typeId = namespaceId.datasetType(clazz.getName());
            boolean hasType = AuthorizationUtil.authorizeAs(authorizingUser, new Callable<Boolean>() {

                @Override
                public Boolean call() throws Exception {
                    return datasetFramework.hasType(typeId);
                }
            });
            if (hasType && !allowDatasetUncheckedUpgrade) {
                return;
            }
            module = new SingleTypeModule(clazz);
        } else {
            throw new IllegalArgumentException(String.format("Cannot use class %s to add dataset module: it must be of type DatasetModule or Dataset", clazz.getName()));
        }
        LOG.info("Adding module: {}", clazz.getName());
        AuthorizationUtil.authorizeAs(authorizingUser, new Callable<Void>() {

            @Override
            public Void call() throws Exception {
                datasetFramework.addModule(moduleId, module, jarLocation);
                return null;
            }
        });
    } catch (ModuleConflictException e) {
        LOG.info("Conflict while deploying module {}: {}", moduleName, e.getMessage());
        throw e;
    }
}