Examples with DatasetTypeId io.cdap.cdap.proto.id.DatasetTypeId used on opensource projects

Example 1 with DatasetTypeId

use of io.cdap.cdap.proto.id.DatasetTypeId in project cdap by caskdata.

the class DatasetModulesDeployer method loadAndDeployModule.

private void loadAndDeployModule(ClassLoader artifactClassLoader, String className, final Location jarLocation, String moduleName, NamespaceId namespaceId, String authorizingUser) throws Exception {
    // note: using app class loader to load module class
    @SuppressWarnings("unchecked") Class<Dataset> clazz = (Class<Dataset>) artifactClassLoader.loadClass(className);
    try {
        // note: we can deploy module or create module from Dataset class
        // note: it seems dangerous to instantiate dataset module here, but this will be fine when we move deploy into
        // isolated user's environment (e.g. separate yarn container)
        final DatasetModuleId moduleId = namespaceId.datasetModule(moduleName);
        final DatasetModule module;
        if (DatasetModule.class.isAssignableFrom(clazz)) {
            module = (DatasetModule) clazz.newInstance();
        } else if (Dataset.class.isAssignableFrom(clazz)) {
            if (systemDatasetFramework.hasSystemType(clazz.getName())) {
                return;
            }
            final DatasetTypeId typeId = namespaceId.datasetType(clazz.getName());
            boolean hasType = AuthorizationUtil.authorizeAs(authorizingUser, new Callable<Boolean>() {

                @Override
                public Boolean call() throws Exception {
                    return datasetFramework.hasType(typeId);
                }
            });
            if (hasType && !allowDatasetUncheckedUpgrade) {
                return;
            }
            module = new SingleTypeModule(clazz);
        } else {
            throw new IllegalArgumentException(String.format("Cannot use class %s to add dataset module: it must be of type DatasetModule or Dataset", clazz.getName()));
        }
        LOG.info("Adding module: {}", clazz.getName());
        AuthorizationUtil.authorizeAs(authorizingUser, new Callable<Void>() {

            @Override
            public Void call() throws Exception {
                datasetFramework.addModule(moduleId, module, jarLocation);
                return null;
            }
        });
    } catch (ModuleConflictException e) {
        LOG.info("Conflict while deploying module {}: {}", moduleName, e.getMessage());
        throw e;
    }
}

Example 2 with DatasetTypeId

use of io.cdap.cdap.proto.id.DatasetTypeId in project cdap by caskdata.

the class DatasetModulesDeployer method deployModules.

/**
 * Deploy the given dataset modules.
 *
 * @param namespaceId namespace to deploy to
 * @param modules the dataset modules to deploy
 * @param jarLocation the location of the jar file containing the modules
 * @param authorizingUser the authorizing user who will be making the call
 * @throws Exception if there was a problem deploying a module
 */
void deployModules(NamespaceId namespaceId, Map<String, String> modules, Location jarLocation, ClassLoader artifactClassLoader, String authorizingUser) throws Exception {
    List<String> implicitModules = new ArrayList<>();
    for (Map.Entry<String, String> moduleEntry : modules.entrySet()) {
        String moduleName = moduleEntry.getKey();
        String typeName = moduleEntry.getValue();
        if (systemDatasetFramework.hasSystemType(typeName)) {
            LOG.info("Not adding dataset type '{}' because it is defined by the system.", typeName);
            continue;
        }
        // Filter out the implicit modules: They have to be deployed last.
        if (moduleName.startsWith(".implicit.")) {
            implicitModules.add(typeName);
            continue;
        }
        loadAndDeployModule(artifactClassLoader, typeName, jarLocation, moduleName, namespaceId, authorizingUser);
    }
    for (String typeName : implicitModules) {
        final DatasetTypeId typeId = namespaceId.datasetType(typeName);
        DatasetTypeMeta typeMeta = AuthorizationUtil.authorizeAs(authorizingUser, new Callable<DatasetTypeMeta>() {

            @Override
            public DatasetTypeMeta call() throws Exception {
                return datasetFramework.getTypeInfo(typeId);
            }
        });
        if (typeMeta != null) {
            String existingModule = Iterables.getLast(typeMeta.getModules()).getName();
            if (modules.containsKey(existingModule)) {
                // it was deployed already as part of one of the explicit deployModule() calls
                continue;
            }
        }
        loadAndDeployModule(artifactClassLoader, typeName, jarLocation, typeName, namespaceId, authorizingUser);
    }
}

Example 3 with DatasetTypeId

use of io.cdap.cdap.proto.id.DatasetTypeId in project cdap by caskdata.

the class DatasetTypeHandlerTest method testNotFound.

@Test
public void testNotFound() throws Exception {
    NamespaceId nonExistent = new NamespaceId("nonExistent");
    HttpResponse response = makeModulesRequest(nonExistent);
    assertNamespaceNotFound(response, nonExistent);
    response = makeTypesRequest(nonExistent);
    assertNamespaceNotFound(response, nonExistent);
    DatasetModuleId datasetModule = nonExistent.datasetModule("module");
    response = makeModuleInfoRequest(datasetModule);
    assertNamespaceNotFound(response, nonExistent);
    DatasetTypeId datasetType = nonExistent.datasetType("type");
    response = makeTypeInfoRequest(datasetType);
    assertNamespaceNotFound(response, nonExistent);
    response = deployModule(datasetModule, TestModule1.class);
    assertNamespaceNotFound(response, nonExistent);
    response = deleteModule(datasetModule);
    assertNamespaceNotFound(response, nonExistent);
    response = deleteModules(nonExistent);
    assertNamespaceNotFound(response, nonExistent);
}

Example 4 with DatasetTypeId

use of io.cdap.cdap.proto.id.DatasetTypeId in project cdap by caskdata.

the class DatasetInstanceService method getTypeInfo.

/**
 * Finds the {@link DatasetTypeMeta} for the specified dataset type name.
 * Search order - first in the specified namespace, then in the 'system' namespace from defaultModules
 *
 * @param namespaceId {@link NamespaceId} for the specified namespace
 * @param typeName the name of the dataset type to search
 * @param byPassCheck a flag which determines whether to check privilege for the dataset type
 * @return {@link DatasetTypeMeta} for the type if found in either the specified namespace or in the system namespace,
 * null otherwise.
 * TODO: This may need to move to a util class eventually
 */
@Nullable
private DatasetTypeMeta getTypeInfo(NamespaceId namespaceId, String typeName, boolean byPassCheck) throws Exception {
    DatasetTypeId datasetTypeId = ConversionHelpers.toDatasetTypeId(namespaceId, typeName);
    try {
        LOG.trace("Retrieving metadata from mds for dataset type {} with authorization: {}", typeName, byPassCheck);
        DatasetTypeMeta meta = byPassCheck ? noAuthDatasetTypeService.getType(datasetTypeId) : authorizationDatasetTypeService.getType(datasetTypeId);
        LOG.trace("Retrieved metadata from mds for dataset type {}", typeName);
        return meta;
    } catch (DatasetTypeNotFoundException | UnauthorizedException e) {
        try {
            // Type not found in the instance's namespace. Now try finding it in the system namespace
            LOG.trace("Retrieving metadata from mds for system dataset type {}", typeName);
            DatasetTypeId systemDatasetTypeId = ConversionHelpers.toDatasetTypeId(NamespaceId.SYSTEM, typeName);
            LOG.trace("Retrieved metadata from mds for system dataset type {}", typeName);
            return noAuthDatasetTypeService.getType(systemDatasetTypeId);
        } catch (DatasetTypeNotFoundException exnWithSystemNS) {
            // if it's not found in system namespace, throw the original exception with the correct namespace
            throw e;
        }
    }
}

Example 5 with DatasetTypeId

use of io.cdap.cdap.proto.id.DatasetTypeId in project cdap by caskdata.

the class DatasetTypeManager method addModule.

/**
 * Add datasets module in a namespace
 *
 * @param datasetModuleId the {@link DatasetModuleId} to add
 * @param className module class
 * @param jarLocation location of the module jar
 * @param force if true, an update will be allowed even if there are conflicts with other modules, or if
 *                     removal of a type would break other modules' dependencies.
 */
public void addModule(final DatasetModuleId datasetModuleId, final String className, final Location jarLocation, final boolean force) throws DatasetModuleConflictException {
    LOG.debug("adding module: {}, className: {}, jarLocation: {}", datasetModuleId, className, jarLocation == null ? "[local]" : jarLocation);
    try {
        TransactionRunners.run(transactionRunner, context -> {
            final DatasetTypeTable datasetTypeTable = DatasetTypeTable.create(context);
            final DatasetInstanceTable datasetInstanceTable = new DatasetInstanceTable(context);
            // 1. get existing module with all its types
            DatasetModuleMeta existing = datasetTypeTable.getModule(datasetModuleId);
            DependencyTrackingRegistry reg;
            // 2. unpack jar and create class loader
            ClassLoaderFolder classLoaderFolder = null;
            DirectoryClassLoader cl = null;
            try {
                // NOTE: if jarLocation is null, we assume that this is a system module, ie. always present in classpath
                if (jarLocation != null) {
                    classLoaderFolder = BundleJarUtil.prepareClassLoaderFolder(jarLocation, () -> Files.createTempDirectory(Files.createDirectories(systemTempPath), datasetModuleId.getEntityName()).toFile());
                    cl = new DirectoryClassLoader(classLoaderFolder.getDir(), cConf.get(Constants.AppFabric.PROGRAM_EXTRA_CLASSPATH), FilterClassLoader.create(getClass().getClassLoader()), "lib");
                }
                reg = new DependencyTrackingRegistry(datasetModuleId, datasetTypeTable, cl, force);
                // 3. register the new module while tracking dependencies.
                // this will fail if a type exists in a different module
                DatasetDefinitionRegistries.register(className, cl, reg);
            } catch (TypeConflictException e) {
                // type conflict from the registry, we want to throw that as is
                throw e;
            } catch (Exception e) {
                LOG.error("Could not instantiate instance of dataset module class {} for module {} using jarLocation {}", className, datasetModuleId, jarLocation);
                throw Throwables.propagate(e);
            } finally {
                // Close the ProgramClassLoader
                Closeables.closeQuietly(cl);
                Closeables.closeQuietly(classLoaderFolder);
            }
            // 4. determine whether any type were removed from the module, and whether any other modules depend on them
            if (existing != null) {
                Set<String> removedTypes = new HashSet<>(existing.getTypes());
                removedTypes.removeAll(reg.getTypes());
                // TODO (CDAP-6294): track dependencies at the type level
                if (!force && !removedTypes.isEmpty() && !existing.getUsedByModules().isEmpty()) {
                    throw new DatasetModuleConflictException(String.format("Cannot update module '%s' to remove types %s: Modules %s may depend on it. Delete them first", datasetModuleId, removedTypes, existing.getUsedByModules()));
                }
                Collection<DatasetSpecification> instances = datasetInstanceTable.getByTypes(datasetModuleId.getParent(), removedTypes);
                if (!instances.isEmpty()) {
                    throw new DatasetModuleConflictException(String.format("Attempt to remove dataset types %s from module '%s' that have existing instances: %s. " + "Delete them first.", removedTypes, datasetModuleId, instances.stream().map(input -> input.getName() + ":" + input.getType()).collect(Collectors.joining(", "))));
                }
            }
            // NOTE: we use set to avoid duplicated dependencies
            // NOTE: we use LinkedHashSet to preserve order in which dependencies must be loaded
            Set<String> moduleDependencies = new LinkedHashSet<String>();
            for (DatasetTypeId usedType : reg.getUsedTypes()) {
                DatasetModuleMeta usedModule = datasetTypeTable.getModuleByType(usedType);
                if (usedModule == null) {
                    throw new IllegalStateException(String.format("Found a null used module for type %s for while adding module %s", usedType, datasetModuleId));
                }
                // adding all used types and the module itself, in this very order to keep the order of loading modules
                // for instantiating a type
                moduleDependencies.addAll(usedModule.getUsesModules());
                boolean added = moduleDependencies.add(usedModule.getName());
                if (added) {
                    // also adding this module as a dependent for all modules it uses
                    usedModule.addUsedByModule(datasetModuleId.getEntityName());
                    datasetTypeTable.writeModule(usedType.getParent(), usedModule);
                }
            }
            URI jarURI = jarLocation == null ? null : jarLocation.toURI();
            DatasetModuleMeta moduleMeta = existing == null ? new DatasetModuleMeta(datasetModuleId.getEntityName(), className, jarURI, reg.getTypes(), Lists.newArrayList(moduleDependencies)) : new DatasetModuleMeta(datasetModuleId.getEntityName(), className, jarURI, reg.getTypes(), Lists.newArrayList(moduleDependencies), Lists.newArrayList(existing.getUsedByModules()));
            datasetTypeTable.writeModule(datasetModuleId.getParent(), moduleMeta);
        });
    } catch (RuntimeException e) {
        for (Throwable cause : Throwables.getCausalChain(e)) {
            if (cause instanceof DatasetModuleConflictException) {
                throw (DatasetModuleConflictException) cause;
            } else if (cause instanceof TypeConflictException) {
                throw new DatasetModuleConflictException(cause.getMessage(), cause);
            }
        }
        throw Throwables.propagate(e);
    } catch (Exception e) {
        LOG.error("Operation failed", e);
        throw Throwables.propagate(e);
    }
}