Search in sources :

Example 1 with DatasetInstanceMDS

use of co.cask.cdap.data2.datafabric.dataset.service.mds.DatasetInstanceMDS in project cdap by caskdata.

the class DatasetInstanceManager method add.

/**
 * Adds dataset instance metadata
 * @param namespaceId the {@link NamespaceId} to add the dataset instance to
 * @param spec {@link DatasetSpecification} of the dataset instance to be added
 */
public void add(final NamespaceId namespaceId, final DatasetSpecification spec) {
    final DatasetInstanceMDS instanceMDS = datasetCache.getDataset(DatasetMetaTableUtil.INSTANCE_TABLE_NAME);
    txExecutorFactory.createExecutor(datasetCache).executeUnchecked(() -> instanceMDS.write(namespaceId, spec));
}
Also used : DatasetInstanceMDS(co.cask.cdap.data2.datafabric.dataset.service.mds.DatasetInstanceMDS)

Example 2 with DatasetInstanceMDS

use of co.cask.cdap.data2.datafabric.dataset.service.mds.DatasetInstanceMDS in project cdap by caskdata.

the class DatasetTypeManager method deleteModule.

/**
 * Deletes specified dataset module
 * @param datasetModuleId {@link DatasetModuleId} of the dataset module to delete
 * @return true if deleted successfully, false if module didn't exist: nothing to delete
 * @throws DatasetModuleConflictException when there are other modules depend on the specified one, in which case
 *         deletion does NOT happen
 */
public boolean deleteModule(final DatasetModuleId datasetModuleId) throws DatasetModuleConflictException {
    LOG.info("Deleting module {}", datasetModuleId);
    try {
        final DatasetTypeMDS datasetTypeMDS = datasetCache.getDataset(DatasetMetaTableUtil.META_TABLE_NAME);
        final DatasetInstanceMDS datasetInstanceMDS = datasetCache.getDataset(DatasetMetaTableUtil.INSTANCE_TABLE_NAME);
        return txExecutorFactory.createExecutor(datasetCache).execute(new Callable<Boolean>() {

            @Override
            public Boolean call() throws DatasetModuleConflictException, IOException {
                final DatasetModuleMeta module = datasetTypeMDS.getModule(datasetModuleId);
                if (module == null) {
                    return false;
                }
                // cannot delete when there's module that uses it
                if (module.getUsedByModules().size() > 0) {
                    String msg = String.format("Cannot delete module %s: other modules depend on it. Delete them first", module);
                    throw new DatasetModuleConflictException(msg);
                }
                Collection<DatasetSpecification> instances = datasetInstanceMDS.getByTypes(datasetModuleId.getParent(), ImmutableSet.copyOf(module.getTypes()));
                // cannot delete when there's instance that uses it
                if (!instances.isEmpty()) {
                    String msg = String.format("Cannot delete module %s: other instances depend on it. Delete them first", module);
                    throw new DatasetModuleConflictException(msg);
                }
                // remove it from "usedBy" from other modules
                for (String usedModuleName : module.getUsesModules()) {
                    DatasetModuleId usedModuleId = new DatasetModuleId(datasetModuleId.getNamespace(), usedModuleName);
                    // not using getModuleWithFallback here because we want to know the namespace in which usedModule was found,
                    // so we can overwrite it in the MDS in the appropriate namespace
                    DatasetModuleMeta usedModule = datasetTypeMDS.getModule(usedModuleId);
                    // if the usedModule is not found in the current namespace, try finding it in the system namespace
                    if (usedModule == null) {
                        usedModuleId = NamespaceId.SYSTEM.datasetModule(usedModuleName);
                        usedModule = datasetTypeMDS.getModule(usedModuleId);
                        Preconditions.checkState(usedModule != null, "Could not find a module %s that the module %s uses.", usedModuleName, datasetModuleId.getEntityName());
                    }
                    usedModule.removeUsedByModule(datasetModuleId.getEntityName());
                    datasetTypeMDS.writeModule(usedModuleId.getParent(), usedModule);
                }
                datasetTypeMDS.deleteModule(datasetModuleId);
                try {
                    // Also delete module jar
                    Location moduleJarLocation = impersonator.doAs(datasetModuleId, new Callable<Location>() {

                        @Override
                        public Location call() throws Exception {
                            return Locations.getLocationFromAbsolutePath(locationFactory, module.getJarLocationPath());
                        }
                    });
                    if (!moduleJarLocation.delete()) {
                        LOG.debug("Could not delete dataset module archive");
                    }
                } catch (Exception e) {
                    // the only checked exception the try-catch throws is IOException
                    Throwables.propagateIfInstanceOf(e, IOException.class);
                    Throwables.propagate(e);
                }
                return true;
            }
        });
    } catch (TransactionFailureException e) {
        if (e.getCause() != null && e.getCause() instanceof DatasetModuleConflictException) {
            throw (DatasetModuleConflictException) e.getCause();
        }
        throw Throwables.propagate(e);
    } catch (Exception e) {
        LOG.error("Operation failed", e);
        throw Throwables.propagate(e);
    }
}
Also used : DatasetInstanceMDS(co.cask.cdap.data2.datafabric.dataset.service.mds.DatasetInstanceMDS) IOException(java.io.IOException) Callable(java.util.concurrent.Callable) TransactionFailureException(org.apache.tephra.TransactionFailureException) TypeConflictException(co.cask.cdap.data2.dataset2.TypeConflictException) IOException(java.io.IOException) DatasetTypeMDS(co.cask.cdap.data2.datafabric.dataset.service.mds.DatasetTypeMDS) DatasetModuleId(co.cask.cdap.proto.id.DatasetModuleId) TransactionFailureException(org.apache.tephra.TransactionFailureException) DatasetModuleMeta(co.cask.cdap.proto.DatasetModuleMeta) Collection(java.util.Collection) Location(org.apache.twill.filesystem.Location)

Example 3 with DatasetInstanceMDS

use of co.cask.cdap.data2.datafabric.dataset.service.mds.DatasetInstanceMDS in project cdap by caskdata.

the class DatasetTypeManager method deleteModules.

/**
 * Deletes all modules in a namespace, other than system.
 * Presumes that the namespace has already been checked to be non-system.
 *
 * @param namespaceId the {@link NamespaceId} to delete modules from.
 */
public void deleteModules(final NamespaceId namespaceId) throws DatasetModuleConflictException {
    Preconditions.checkArgument(namespaceId != null && !NamespaceId.SYSTEM.equals(namespaceId), "Cannot delete modules from system namespace");
    LOG.info("Deleting all modules from namespace {}", namespaceId);
    try {
        final DatasetTypeMDS datasetTypeMDS = datasetCache.getDataset(DatasetMetaTableUtil.META_TABLE_NAME);
        final DatasetInstanceMDS datasetInstanceMDS = datasetCache.getDataset(DatasetMetaTableUtil.INSTANCE_TABLE_NAME);
        txExecutorFactory.createExecutor(datasetCache).execute(new TransactionExecutor.Subroutine() {

            @Override
            public void apply() throws DatasetModuleConflictException, IOException {
                final Set<String> typesToDelete = new HashSet<String>();
                final List<Location> moduleLocations = new ArrayList<>();
                final Collection<DatasetModuleMeta> modules = datasetTypeMDS.getModules(namespaceId);
                try {
                    impersonator.doAs(namespaceId, new Callable<Void>() {

                        @Override
                        public Void call() throws Exception {
                            for (DatasetModuleMeta module : modules) {
                                typesToDelete.addAll(module.getTypes());
                                moduleLocations.add(Locations.getLocationFromAbsolutePath(locationFactory, module.getJarLocationPath()));
                            }
                            return null;
                        }
                    });
                } catch (Exception e) {
                    // the callable throws no checked exceptions
                    Throwables.propagate(e);
                }
                // check if there are any instances that use types of these modules?
                Collection<DatasetSpecification> instances = datasetInstanceMDS.getByTypes(namespaceId, typesToDelete);
                // cannot delete when there's instance that uses it
                if (!instances.isEmpty()) {
                    throw new DatasetModuleConflictException("Cannot delete all modules: existing dataset instances depend on it. Delete them first");
                }
                datasetTypeMDS.deleteModules(namespaceId);
                // Delete module locations
                for (Location moduleLocation : moduleLocations) {
                    if (!moduleLocation.delete()) {
                        LOG.debug("Could not delete dataset module archive - {}", moduleLocation);
                    }
                }
            }
        });
    } catch (TransactionFailureException e) {
        if (e.getCause() != null && e.getCause() instanceof DatasetModuleConflictException) {
            throw (DatasetModuleConflictException) e.getCause();
        }
        LOG.error("Failed to delete all modules from namespace {}", namespaceId);
        throw Throwables.propagate(e);
    } catch (Exception e) {
        LOG.error("Operation failed", e);
        throw Throwables.propagate(e);
    }
}
Also used : ImmutableSet(com.google.common.collect.ImmutableSet) Set(java.util.Set) HashSet(java.util.HashSet) LinkedHashSet(java.util.LinkedHashSet) DatasetInstanceMDS(co.cask.cdap.data2.datafabric.dataset.service.mds.DatasetInstanceMDS) TransactionExecutor(org.apache.tephra.TransactionExecutor) IOException(java.io.IOException) Callable(java.util.concurrent.Callable) TransactionFailureException(org.apache.tephra.TransactionFailureException) TypeConflictException(co.cask.cdap.data2.dataset2.TypeConflictException) IOException(java.io.IOException) DatasetTypeMDS(co.cask.cdap.data2.datafabric.dataset.service.mds.DatasetTypeMDS) TransactionFailureException(org.apache.tephra.TransactionFailureException) DatasetModuleMeta(co.cask.cdap.proto.DatasetModuleMeta) Collection(java.util.Collection) List(java.util.List) ArrayList(java.util.ArrayList) Location(org.apache.twill.filesystem.Location)

Example 4 with DatasetInstanceMDS

use of co.cask.cdap.data2.datafabric.dataset.service.mds.DatasetInstanceMDS in project cdap by caskdata.

the class DatasetTypeManager method addModule.

/**
 * Add datasets module in a namespace
 *
 * @param datasetModuleId the {@link DatasetModuleId} to add
 * @param className module class
 * @param jarLocation location of the module jar
 * @param force if true, an update will be allowed even if there are conflicts with other modules, or if
 *                     removal of a type would break other modules' dependencies.
 */
public void addModule(final DatasetModuleId datasetModuleId, final String className, final Location jarLocation, final boolean force) throws DatasetModuleConflictException {
    LOG.debug("adding module: {}, className: {}, jarLocation: {}", datasetModuleId, className, jarLocation == null ? "[local]" : jarLocation);
    try {
        final DatasetTypeMDS datasetTypeMDS = datasetCache.getDataset(DatasetMetaTableUtil.META_TABLE_NAME);
        final DatasetInstanceMDS datasetInstanceMDS = datasetCache.getDataset(DatasetMetaTableUtil.INSTANCE_TABLE_NAME);
        txExecutorFactory.createExecutor(datasetCache).execute(new TransactionExecutor.Subroutine() {

            @Override
            public void apply() throws Exception {
                // 1. get existing module with all its types
                DatasetModuleMeta existing = datasetTypeMDS.getModule(datasetModuleId);
                DependencyTrackingRegistry reg;
                // 2. unpack jar and create class loader
                File unpackedLocation = Files.createTempDirectory(Files.createDirectories(systemTempPath), datasetModuleId.getEntityName()).toFile();
                DirectoryClassLoader cl = null;
                try {
                    // NOTE: if jarLocation is null, we assume that this is a system module, ie. always present in classpath
                    if (jarLocation != null) {
                        BundleJarUtil.unJar(jarLocation, unpackedLocation);
                        cl = new DirectoryClassLoader(unpackedLocation, cConf.get(Constants.AppFabric.PROGRAM_EXTRA_CLASSPATH), FilterClassLoader.create(getClass().getClassLoader()), "lib");
                    }
                    reg = new DependencyTrackingRegistry(datasetModuleId, datasetTypeMDS, cl, force);
                    // 3. register the new module while tracking dependencies.
                    // this will fail if a type exists in a different module
                    DatasetDefinitionRegistries.register(className, cl, reg);
                } catch (TypeConflictException e) {
                    // type conflict from the registry, we want to throw that as is
                    throw e;
                } catch (Exception e) {
                    LOG.error("Could not instantiate instance of dataset module class {} for module {} using jarLocation {}", className, datasetModuleId, jarLocation);
                    throw Throwables.propagate(e);
                } finally {
                    if (cl != null) {
                        // Close the ProgramClassLoader
                        Closeables.closeQuietly(cl);
                    }
                    try {
                        DirUtils.deleteDirectoryContents(unpackedLocation);
                    } catch (IOException e) {
                        LOG.warn("Failed to delete directory {}", unpackedLocation, e);
                    }
                }
                // 4. determine whether any type were removed from the module, and whether any other modules depend on them
                if (existing != null) {
                    Set<String> removedTypes = new HashSet<>(existing.getTypes());
                    removedTypes.removeAll(reg.getTypes());
                    // TODO (CDAP-6294): track dependencies at the type level
                    if (!force && !removedTypes.isEmpty() && !existing.getUsedByModules().isEmpty()) {
                        throw new DatasetModuleConflictException(String.format("Cannot update module '%s' to remove types %s: Modules %s may depend on it. Delete them first", datasetModuleId, removedTypes, existing.getUsedByModules()));
                    }
                    Collection<DatasetSpecification> instances = datasetInstanceMDS.getByTypes(datasetModuleId.getParent(), removedTypes);
                    if (!instances.isEmpty()) {
                        throw new DatasetModuleConflictException(String.format("Attempt to remove dataset types %s from module '%s' that have existing instances: %s. " + "Delete them first.", removedTypes, datasetModuleId, Iterables.toString(Iterables.transform(instances, new Function<DatasetSpecification, String>() {

                            @Nullable
                            @Override
                            public String apply(@Nullable DatasetSpecification input) {
                                return input.getName() + ":" + input.getType();
                            }
                        }))));
                    }
                }
                // NOTE: we use set to avoid duplicated dependencies
                // NOTE: we use LinkedHashSet to preserve order in which dependencies must be loaded
                Set<String> moduleDependencies = new LinkedHashSet<String>();
                for (DatasetTypeId usedType : reg.getUsedTypes()) {
                    DatasetModuleMeta usedModule = datasetTypeMDS.getModuleByType(usedType);
                    Preconditions.checkState(usedModule != null, String.format("Found a null used module for type %s for while adding module %s", usedType, datasetModuleId));
                    // adding all used types and the module itself, in this very order to keep the order of loading modules
                    // for instantiating a type
                    moduleDependencies.addAll(usedModule.getUsesModules());
                    boolean added = moduleDependencies.add(usedModule.getName());
                    if (added) {
                        // also adding this module as a dependent for all modules it uses
                        usedModule.addUsedByModule(datasetModuleId.getEntityName());
                        datasetTypeMDS.writeModule(usedType.getParent(), usedModule);
                    }
                }
                URI jarURI = jarLocation == null ? null : jarLocation.toURI();
                DatasetModuleMeta moduleMeta = existing == null ? new DatasetModuleMeta(datasetModuleId.getEntityName(), className, jarURI, reg.getTypes(), Lists.newArrayList(moduleDependencies)) : new DatasetModuleMeta(datasetModuleId.getEntityName(), className, jarURI, reg.getTypes(), Lists.newArrayList(moduleDependencies), Lists.newArrayList(existing.getUsedByModules()));
                datasetTypeMDS.writeModule(datasetModuleId.getParent(), moduleMeta);
            }
        });
    } catch (TransactionFailureException e) {
        Throwable cause = e.getCause();
        if (cause != null) {
            if (cause instanceof DatasetModuleConflictException) {
                throw (DatasetModuleConflictException) cause;
            } else if (cause instanceof TypeConflictException) {
                throw new DatasetModuleConflictException(cause.getMessage(), cause);
            }
        }
        throw Throwables.propagate(e);
    } catch (Exception e) {
        LOG.error("Operation failed", e);
        throw Throwables.propagate(e);
    }
}
Also used : ImmutableSet(com.google.common.collect.ImmutableSet) Set(java.util.Set) HashSet(java.util.HashSet) LinkedHashSet(java.util.LinkedHashSet) DatasetTypeId(co.cask.cdap.proto.id.DatasetTypeId) URI(java.net.URI) DatasetModuleMeta(co.cask.cdap.proto.DatasetModuleMeta) DirectoryClassLoader(co.cask.cdap.common.lang.DirectoryClassLoader) TypeConflictException(co.cask.cdap.data2.dataset2.TypeConflictException) DatasetInstanceMDS(co.cask.cdap.data2.datafabric.dataset.service.mds.DatasetInstanceMDS) DatasetSpecification(co.cask.cdap.api.dataset.DatasetSpecification) TransactionExecutor(org.apache.tephra.TransactionExecutor) IOException(java.io.IOException) TransactionFailureException(org.apache.tephra.TransactionFailureException) TypeConflictException(co.cask.cdap.data2.dataset2.TypeConflictException) IOException(java.io.IOException) DatasetTypeMDS(co.cask.cdap.data2.datafabric.dataset.service.mds.DatasetTypeMDS) TransactionFailureException(org.apache.tephra.TransactionFailureException) Collection(java.util.Collection) File(java.io.File) Nullable(javax.annotation.Nullable)

Aggregations

DatasetInstanceMDS (co.cask.cdap.data2.datafabric.dataset.service.mds.DatasetInstanceMDS)4 DatasetTypeMDS (co.cask.cdap.data2.datafabric.dataset.service.mds.DatasetTypeMDS)3 TypeConflictException (co.cask.cdap.data2.dataset2.TypeConflictException)3 DatasetModuleMeta (co.cask.cdap.proto.DatasetModuleMeta)3 IOException (java.io.IOException)3 Collection (java.util.Collection)3 TransactionFailureException (org.apache.tephra.TransactionFailureException)3 ImmutableSet (com.google.common.collect.ImmutableSet)2 HashSet (java.util.HashSet)2 LinkedHashSet (java.util.LinkedHashSet)2 Set (java.util.Set)2 Callable (java.util.concurrent.Callable)2 TransactionExecutor (org.apache.tephra.TransactionExecutor)2 Location (org.apache.twill.filesystem.Location)2 DatasetSpecification (co.cask.cdap.api.dataset.DatasetSpecification)1 DirectoryClassLoader (co.cask.cdap.common.lang.DirectoryClassLoader)1 DatasetModuleId (co.cask.cdap.proto.id.DatasetModuleId)1 DatasetTypeId (co.cask.cdap.proto.id.DatasetTypeId)1 File (java.io.File)1 URI (java.net.URI)1