Examples with DatasetModuleId io.cdap.cdap.proto.id.DatasetModuleId used on opensource projects

Example 36 with DatasetModuleId

use of io.cdap.cdap.proto.id.DatasetModuleId in project cdap by cdapio.

the class DatasetTypeTable method writeModule.

public void writeModule(NamespaceId namespaceId, DatasetModuleMeta moduleMeta) throws IOException {
    DatasetModuleId datasetModuleId = namespaceId.datasetModule(moduleMeta.getName());
    DatasetModuleMeta existing = getModule(datasetModuleId);
    List<Field<?>> fields = getModuleKey(namespaceId.getEntityName(), moduleMeta.getName());
    fields.add(Fields.stringField(StoreDefinition.DatasetTypeStore.DATASET_METADATA_FIELD, GSON.toJson(moduleMeta)));
    getModuleTable().upsert(fields);
    for (String type : moduleMeta.getTypes()) {
        writeTypeToModuleMapping(namespaceId.datasetType(type), datasetModuleId);
    }
    if (existing != null) {
        Set<String> removed = new HashSet<>(existing.getTypes());
        removed.removeAll(moduleMeta.getTypes());
        for (String type : removed) {
            getTypeTable().deleteAll(Range.singleton(getTypeKey(datasetModuleId.getNamespace(), type)));
        }
    }
}

Example 37 with DatasetModuleId

use of io.cdap.cdap.proto.id.DatasetModuleId in project cdap by cdapio.

the class DatasetTypeManager method addModule.

/**
 * Add datasets module in a namespace
 *
 * @param datasetModuleId the {@link DatasetModuleId} to add
 * @param className module class
 * @param jarLocation location of the module jar
 * @param force if true, an update will be allowed even if there are conflicts with other modules, or if
 *                     removal of a type would break other modules' dependencies.
 */
public void addModule(final DatasetModuleId datasetModuleId, final String className, final Location jarLocation, final boolean force) throws DatasetModuleConflictException {
    LOG.debug("adding module: {}, className: {}, jarLocation: {}", datasetModuleId, className, jarLocation == null ? "[local]" : jarLocation);
    try {
        TransactionRunners.run(transactionRunner, context -> {
            final DatasetTypeTable datasetTypeTable = DatasetTypeTable.create(context);
            final DatasetInstanceTable datasetInstanceTable = new DatasetInstanceTable(context);
            // 1. get existing module with all its types
            DatasetModuleMeta existing = datasetTypeTable.getModule(datasetModuleId);
            DependencyTrackingRegistry reg;
            // 2. unpack jar and create class loader
            ClassLoaderFolder classLoaderFolder = null;
            DirectoryClassLoader cl = null;
            try {
                // NOTE: if jarLocation is null, we assume that this is a system module, ie. always present in classpath
                if (jarLocation != null) {
                    classLoaderFolder = BundleJarUtil.prepareClassLoaderFolder(jarLocation, () -> Files.createTempDirectory(Files.createDirectories(systemTempPath), datasetModuleId.getEntityName()).toFile());
                    cl = new DirectoryClassLoader(classLoaderFolder.getDir(), cConf.get(Constants.AppFabric.PROGRAM_EXTRA_CLASSPATH), FilterClassLoader.create(getClass().getClassLoader()), "lib");
                }
                reg = new DependencyTrackingRegistry(datasetModuleId, datasetTypeTable, cl, force);
                // 3. register the new module while tracking dependencies.
                // this will fail if a type exists in a different module
                DatasetDefinitionRegistries.register(className, cl, reg);
            } catch (TypeConflictException e) {
                // type conflict from the registry, we want to throw that as is
                throw e;
            } catch (Exception e) {
                LOG.error("Could not instantiate instance of dataset module class {} for module {} using jarLocation {}", className, datasetModuleId, jarLocation);
                throw Throwables.propagate(e);
            } finally {
                // Close the ProgramClassLoader
                Closeables.closeQuietly(cl);
                Closeables.closeQuietly(classLoaderFolder);
            }
            // 4. determine whether any type were removed from the module, and whether any other modules depend on them
            if (existing != null) {
                Set<String> removedTypes = new HashSet<>(existing.getTypes());
                removedTypes.removeAll(reg.getTypes());
                // TODO (CDAP-6294): track dependencies at the type level
                if (!force && !removedTypes.isEmpty() && !existing.getUsedByModules().isEmpty()) {
                    throw new DatasetModuleConflictException(String.format("Cannot update module '%s' to remove types %s: Modules %s may depend on it. Delete them first", datasetModuleId, removedTypes, existing.getUsedByModules()));
                }
                Collection<DatasetSpecification> instances = datasetInstanceTable.getByTypes(datasetModuleId.getParent(), removedTypes);
                if (!instances.isEmpty()) {
                    throw new DatasetModuleConflictException(String.format("Attempt to remove dataset types %s from module '%s' that have existing instances: %s. " + "Delete them first.", removedTypes, datasetModuleId, instances.stream().map(input -> input.getName() + ":" + input.getType()).collect(Collectors.joining(", "))));
                }
            }
            // NOTE: we use set to avoid duplicated dependencies
            // NOTE: we use LinkedHashSet to preserve order in which dependencies must be loaded
            Set<String> moduleDependencies = new LinkedHashSet<String>();
            for (DatasetTypeId usedType : reg.getUsedTypes()) {
                DatasetModuleMeta usedModule = datasetTypeTable.getModuleByType(usedType);
                if (usedModule == null) {
                    throw new IllegalStateException(String.format("Found a null used module for type %s for while adding module %s", usedType, datasetModuleId));
                }
                // adding all used types and the module itself, in this very order to keep the order of loading modules
                // for instantiating a type
                moduleDependencies.addAll(usedModule.getUsesModules());
                boolean added = moduleDependencies.add(usedModule.getName());
                if (added) {
                    // also adding this module as a dependent for all modules it uses
                    usedModule.addUsedByModule(datasetModuleId.getEntityName());
                    datasetTypeTable.writeModule(usedType.getParent(), usedModule);
                }
            }
            URI jarURI = jarLocation == null ? null : jarLocation.toURI();
            DatasetModuleMeta moduleMeta = existing == null ? new DatasetModuleMeta(datasetModuleId.getEntityName(), className, jarURI, reg.getTypes(), Lists.newArrayList(moduleDependencies)) : new DatasetModuleMeta(datasetModuleId.getEntityName(), className, jarURI, reg.getTypes(), Lists.newArrayList(moduleDependencies), Lists.newArrayList(existing.getUsedByModules()));
            datasetTypeTable.writeModule(datasetModuleId.getParent(), moduleMeta);
        });
    } catch (RuntimeException e) {
        for (Throwable cause : Throwables.getCausalChain(e)) {
            if (cause instanceof DatasetModuleConflictException) {
                throw (DatasetModuleConflictException) cause;
            } else if (cause instanceof TypeConflictException) {
                throw new DatasetModuleConflictException(cause.getMessage(), cause);
            }
        }
        throw Throwables.propagate(e);
    } catch (Exception e) {
        LOG.error("Operation failed", e);
        throw Throwables.propagate(e);
    }
}

Example 38 with DatasetModuleId

use of io.cdap.cdap.proto.id.DatasetModuleId in project cdap by cdapio.

the class DatasetTypeManager method deleteModule.

/**
 * Deletes specified dataset module
 * @param datasetModuleId {@link DatasetModuleId} of the dataset module to delete
 * @return true if deleted successfully, false if module didn't exist: nothing to delete
 * @throws DatasetModuleConflictException when there are other modules depend on the specified one, in which case
 *         deletion does NOT happen
 */
public boolean deleteModule(final DatasetModuleId datasetModuleId) throws DatasetModuleConflictException {
    LOG.info("Deleting module {}", datasetModuleId);
    try {
        return TransactionRunners.run(transactionRunner, context -> {
            final DatasetTypeTable datasetTypeTable = DatasetTypeTable.create(context);
            final DatasetInstanceTable datasetInstanceTable = new DatasetInstanceTable(context);
            final DatasetModuleMeta module = datasetTypeTable.getModule(datasetModuleId);
            if (module == null) {
                return false;
            }
            // cannot delete when there's module that uses it
            if (module.getUsedByModules().size() > 0) {
                String msg = String.format("Cannot delete module %s: other modules depend on it. Delete them first", module);
                throw new DatasetModuleConflictException(msg);
            }
            Collection<DatasetSpecification> instances = datasetInstanceTable.getByTypes(datasetModuleId.getParent(), ImmutableSet.copyOf(module.getTypes()));
            // cannot delete when there's instance that uses it
            if (!instances.isEmpty()) {
                String msg = String.format("Cannot delete module %s: other instances depend on it. Delete them first", module);
                throw new DatasetModuleConflictException(msg);
            }
            // remove it from "usedBy" from other modules
            for (String usedModuleName : module.getUsesModules()) {
                DatasetModuleId usedModuleId = new DatasetModuleId(datasetModuleId.getNamespace(), usedModuleName);
                // not using getModuleWithFallback here because we want to know the namespace in which usedModule was found,
                // so we can overwrite it in the MDS in the appropriate namespace
                DatasetModuleMeta usedModule = datasetTypeTable.getModule(usedModuleId);
                // if the usedModule is not found in the current namespace, try finding it in the system namespace
                if (usedModule == null) {
                    usedModuleId = NamespaceId.SYSTEM.datasetModule(usedModuleName);
                    usedModule = datasetTypeTable.getModule(usedModuleId);
                    Preconditions.checkState(usedModule != null, "Could not find a module %s that the module %s uses.", usedModuleName, datasetModuleId.getEntityName());
                }
                usedModule.removeUsedByModule(datasetModuleId.getEntityName());
                datasetTypeTable.writeModule(usedModuleId.getParent(), usedModule);
            }
            datasetTypeTable.deleteModule(datasetModuleId);
            try {
                // Also delete module jar
                Location moduleJarLocation = impersonator.doAs(datasetModuleId, () -> Locations.getLocationFromAbsolutePath(locationFactory, module.getJarLocationPath()));
                if (!moduleJarLocation.delete()) {
                    LOG.debug("Could not delete dataset module archive");
                }
            } catch (Exception e) {
                // the only checked exception the try-catch throws is IOException
                Throwables.propagateIfInstanceOf(e, IOException.class);
                throw Throwables.propagate(e);
            }
            return true;
        });
    } catch (RuntimeException e) {
        for (Throwable cause : Throwables.getCausalChain(e)) {
            if (cause instanceof DatasetModuleConflictException) {
                throw (DatasetModuleConflictException) cause;
            }
        }
        throw Throwables.propagate(e);
    } catch (Exception e) {
        LOG.error("Operation failed", e);
        throw Throwables.propagate(e);
    }
}

Example 39 with DatasetModuleId

use of io.cdap.cdap.proto.id.DatasetModuleId in project cdap by cdapio.

the class GenerateClientUsageExample method datasetModuleClient.

public void datasetModuleClient() throws Exception {
    // Construct the client used to interact with CDAP
    DatasetModuleClient datasetModuleClient = new DatasetModuleClient(clientConfig);
    // Add a dataset module
    File moduleJarFile = createAppJarFile(SomeDatasetModule.class);
    DatasetModuleId datasetModuleId = NamespaceId.DEFAULT.datasetModule("someDatasetModule");
    datasetModuleClient.add(datasetModuleId, SomeDatasetModule.class.getName(), moduleJarFile);
    // Fetch the dataset module information
    DatasetModuleMeta datasetModuleMeta = datasetModuleClient.get(datasetModuleId);
    // Delete all dataset modules
    datasetModuleClient.deleteAll(NamespaceId.DEFAULT);
}

Example 40 with DatasetModuleId

use of io.cdap.cdap.proto.id.DatasetModuleId in project cdap by cdapio.

the class DeleteDatasetModuleCommand method perform.

@Override
public void perform(Arguments arguments, PrintStream output) throws Exception {
    DatasetModuleId module = cliConfig.getCurrentNamespace().datasetModule(arguments.get(ArgumentName.DATASET_MODULE.toString()));
    datasetClient.delete(module);
    output.printf("Successfully deleted dataset module '%s'\n", module.getEntityName());
}