Search in sources :

Example 1 with DatasetModuleConflictException

use of co.cask.cdap.data2.datafabric.dataset.type.DatasetModuleConflictException in project cdap by caskdata.

the class DatasetTypeService method deployExtensionModules.

private void deployExtensionModules() {
    // adding any defined extension modules to be available in dataset manager service
    for (Map.Entry<String, DatasetModule> module : extensionModules.entrySet()) {
        try {
            // NOTE: we assume extension modules are always in classpath, hence passing null for jar location
            // NOTE: we add extension modules in the system namespace
            DatasetModuleId theModule = NamespaceId.SYSTEM.datasetModule(module.getKey());
            typeManager.addModule(theModule, module.getValue().getClass().getName(), null, false);
            grantAllPrivilegesOnModule(theModule, authenticationContext.getPrincipal());
        } catch (DatasetModuleConflictException e) {
            // perfectly fine: we need to add the modules only the very first time service is started
            LOG.debug("Not adding {} extension module: it already exists", module.getKey());
        } catch (Throwable th) {
            LOG.error("Failed to add {} extension module. Aborting.", module.getKey(), th);
            throw Throwables.propagate(th);
        }
    }
}
Also used : DatasetModuleId(co.cask.cdap.proto.id.DatasetModuleId) DatasetModuleConflictException(co.cask.cdap.data2.datafabric.dataset.type.DatasetModuleConflictException) Map(java.util.Map) ImmutableMap(com.google.common.collect.ImmutableMap) LinkedHashMap(java.util.LinkedHashMap) DatasetModule(co.cask.cdap.api.dataset.module.DatasetModule)

Example 2 with DatasetModuleConflictException

use of co.cask.cdap.data2.datafabric.dataset.type.DatasetModuleConflictException in project cdap by caskdata.

the class DefaultDatasetTypeService method deployDefaultModules.

private void deployDefaultModules() throws Exception {
    // adding default modules to be available in dataset manager service
    for (Map.Entry<String, DatasetModule> module : defaultModules.entrySet()) {
        try {
            // NOTE: we assume default modules are always in classpath, hence passing null for jar location
            // NOTE: we add default modules in the system namespace
            DatasetModuleId defaultModule = NamespaceId.SYSTEM.datasetModule(module.getKey());
            typeManager.addModule(defaultModule, module.getValue().getClass().getName(), null, false);
        } catch (DatasetModuleConflictException e) {
            // perfectly fine: we need to add default modules only the very first time service is started
            LOG.debug("Not adding {} module: it already exists", module.getKey());
        } catch (Throwable th) {
            LOG.error("Failed to add {} module. Aborting.", module.getKey(), th);
            throw Throwables.propagate(th);
        }
    }
}
Also used : DatasetModuleId(co.cask.cdap.proto.id.DatasetModuleId) DatasetModuleConflictException(co.cask.cdap.data2.datafabric.dataset.type.DatasetModuleConflictException) Map(java.util.Map) ImmutableMap(com.google.common.collect.ImmutableMap) LinkedHashMap(java.util.LinkedHashMap) DatasetModule(co.cask.cdap.api.dataset.module.DatasetModule)

Example 3 with DatasetModuleConflictException

use of co.cask.cdap.data2.datafabric.dataset.type.DatasetModuleConflictException in project cdap by caskdata.

the class DatasetTypeManager method deleteModule.

/**
 * Deletes specified dataset module
 * @param datasetModuleId {@link DatasetModuleId} of the dataset module to delete
 * @return true if deleted successfully, false if module didn't exist: nothing to delete
 * @throws DatasetModuleConflictException when there are other modules depend on the specified one, in which case
 *         deletion does NOT happen
 */
public boolean deleteModule(final DatasetModuleId datasetModuleId) throws DatasetModuleConflictException {
    LOG.info("Deleting module {}", datasetModuleId);
    try {
        final DatasetTypeMDS datasetTypeMDS = datasetCache.getDataset(DatasetMetaTableUtil.META_TABLE_NAME);
        final DatasetInstanceMDS datasetInstanceMDS = datasetCache.getDataset(DatasetMetaTableUtil.INSTANCE_TABLE_NAME);
        return txExecutorFactory.createExecutor(datasetCache).execute(new Callable<Boolean>() {

            @Override
            public Boolean call() throws DatasetModuleConflictException, IOException {
                final DatasetModuleMeta module = datasetTypeMDS.getModule(datasetModuleId);
                if (module == null) {
                    return false;
                }
                // cannot delete when there's module that uses it
                if (module.getUsedByModules().size() > 0) {
                    String msg = String.format("Cannot delete module %s: other modules depend on it. Delete them first", module);
                    throw new DatasetModuleConflictException(msg);
                }
                Collection<DatasetSpecification> instances = datasetInstanceMDS.getByTypes(datasetModuleId.getParent(), ImmutableSet.copyOf(module.getTypes()));
                // cannot delete when there's instance that uses it
                if (!instances.isEmpty()) {
                    String msg = String.format("Cannot delete module %s: other instances depend on it. Delete them first", module);
                    throw new DatasetModuleConflictException(msg);
                }
                // remove it from "usedBy" from other modules
                for (String usedModuleName : module.getUsesModules()) {
                    DatasetModuleId usedModuleId = new DatasetModuleId(datasetModuleId.getNamespace(), usedModuleName);
                    // not using getModuleWithFallback here because we want to know the namespace in which usedModule was found,
                    // so we can overwrite it in the MDS in the appropriate namespace
                    DatasetModuleMeta usedModule = datasetTypeMDS.getModule(usedModuleId);
                    // if the usedModule is not found in the current namespace, try finding it in the system namespace
                    if (usedModule == null) {
                        usedModuleId = NamespaceId.SYSTEM.datasetModule(usedModuleName);
                        usedModule = datasetTypeMDS.getModule(usedModuleId);
                        Preconditions.checkState(usedModule != null, "Could not find a module %s that the module %s uses.", usedModuleName, datasetModuleId.getEntityName());
                    }
                    usedModule.removeUsedByModule(datasetModuleId.getEntityName());
                    datasetTypeMDS.writeModule(usedModuleId.getParent(), usedModule);
                }
                datasetTypeMDS.deleteModule(datasetModuleId);
                try {
                    // Also delete module jar
                    Location moduleJarLocation = impersonator.doAs(datasetModuleId, new Callable<Location>() {

                        @Override
                        public Location call() throws Exception {
                            return Locations.getLocationFromAbsolutePath(locationFactory, module.getJarLocationPath());
                        }
                    });
                    if (!moduleJarLocation.delete()) {
                        LOG.debug("Could not delete dataset module archive");
                    }
                } catch (Exception e) {
                    // the only checked exception the try-catch throws is IOException
                    Throwables.propagateIfInstanceOf(e, IOException.class);
                    Throwables.propagate(e);
                }
                return true;
            }
        });
    } catch (TransactionFailureException e) {
        if (e.getCause() != null && e.getCause() instanceof DatasetModuleConflictException) {
            throw (DatasetModuleConflictException) e.getCause();
        }
        throw Throwables.propagate(e);
    } catch (Exception e) {
        LOG.error("Operation failed", e);
        throw Throwables.propagate(e);
    }
}
Also used : DatasetInstanceMDS(co.cask.cdap.data2.datafabric.dataset.service.mds.DatasetInstanceMDS) IOException(java.io.IOException) Callable(java.util.concurrent.Callable) TransactionFailureException(org.apache.tephra.TransactionFailureException) TypeConflictException(co.cask.cdap.data2.dataset2.TypeConflictException) IOException(java.io.IOException) DatasetTypeMDS(co.cask.cdap.data2.datafabric.dataset.service.mds.DatasetTypeMDS) DatasetModuleId(co.cask.cdap.proto.id.DatasetModuleId) TransactionFailureException(org.apache.tephra.TransactionFailureException) DatasetModuleMeta(co.cask.cdap.proto.DatasetModuleMeta) Collection(java.util.Collection) Location(org.apache.twill.filesystem.Location)

Example 4 with DatasetModuleConflictException

use of co.cask.cdap.data2.datafabric.dataset.type.DatasetModuleConflictException in project cdap by caskdata.

the class DatasetTypeManager method deleteModules.

/**
 * Deletes all modules in a namespace, other than system.
 * Presumes that the namespace has already been checked to be non-system.
 *
 * @param namespaceId the {@link NamespaceId} to delete modules from.
 */
public void deleteModules(final NamespaceId namespaceId) throws DatasetModuleConflictException {
    Preconditions.checkArgument(namespaceId != null && !NamespaceId.SYSTEM.equals(namespaceId), "Cannot delete modules from system namespace");
    LOG.info("Deleting all modules from namespace {}", namespaceId);
    try {
        final DatasetTypeMDS datasetTypeMDS = datasetCache.getDataset(DatasetMetaTableUtil.META_TABLE_NAME);
        final DatasetInstanceMDS datasetInstanceMDS = datasetCache.getDataset(DatasetMetaTableUtil.INSTANCE_TABLE_NAME);
        txExecutorFactory.createExecutor(datasetCache).execute(new TransactionExecutor.Subroutine() {

            @Override
            public void apply() throws DatasetModuleConflictException, IOException {
                final Set<String> typesToDelete = new HashSet<String>();
                final List<Location> moduleLocations = new ArrayList<>();
                final Collection<DatasetModuleMeta> modules = datasetTypeMDS.getModules(namespaceId);
                try {
                    impersonator.doAs(namespaceId, new Callable<Void>() {

                        @Override
                        public Void call() throws Exception {
                            for (DatasetModuleMeta module : modules) {
                                typesToDelete.addAll(module.getTypes());
                                moduleLocations.add(Locations.getLocationFromAbsolutePath(locationFactory, module.getJarLocationPath()));
                            }
                            return null;
                        }
                    });
                } catch (Exception e) {
                    // the callable throws no checked exceptions
                    Throwables.propagate(e);
                }
                // check if there are any instances that use types of these modules?
                Collection<DatasetSpecification> instances = datasetInstanceMDS.getByTypes(namespaceId, typesToDelete);
                // cannot delete when there's instance that uses it
                if (!instances.isEmpty()) {
                    throw new DatasetModuleConflictException("Cannot delete all modules: existing dataset instances depend on it. Delete them first");
                }
                datasetTypeMDS.deleteModules(namespaceId);
                // Delete module locations
                for (Location moduleLocation : moduleLocations) {
                    if (!moduleLocation.delete()) {
                        LOG.debug("Could not delete dataset module archive - {}", moduleLocation);
                    }
                }
            }
        });
    } catch (TransactionFailureException e) {
        if (e.getCause() != null && e.getCause() instanceof DatasetModuleConflictException) {
            throw (DatasetModuleConflictException) e.getCause();
        }
        LOG.error("Failed to delete all modules from namespace {}", namespaceId);
        throw Throwables.propagate(e);
    } catch (Exception e) {
        LOG.error("Operation failed", e);
        throw Throwables.propagate(e);
    }
}
Also used : ImmutableSet(com.google.common.collect.ImmutableSet) Set(java.util.Set) HashSet(java.util.HashSet) LinkedHashSet(java.util.LinkedHashSet) DatasetInstanceMDS(co.cask.cdap.data2.datafabric.dataset.service.mds.DatasetInstanceMDS) TransactionExecutor(org.apache.tephra.TransactionExecutor) IOException(java.io.IOException) Callable(java.util.concurrent.Callable) TransactionFailureException(org.apache.tephra.TransactionFailureException) TypeConflictException(co.cask.cdap.data2.dataset2.TypeConflictException) IOException(java.io.IOException) DatasetTypeMDS(co.cask.cdap.data2.datafabric.dataset.service.mds.DatasetTypeMDS) TransactionFailureException(org.apache.tephra.TransactionFailureException) DatasetModuleMeta(co.cask.cdap.proto.DatasetModuleMeta) Collection(java.util.Collection) List(java.util.List) ArrayList(java.util.ArrayList) Location(org.apache.twill.filesystem.Location)

Example 5 with DatasetModuleConflictException

use of co.cask.cdap.data2.datafabric.dataset.type.DatasetModuleConflictException in project cdap by caskdata.

the class DatasetTypeService method deployDefaultModules.

private void deployDefaultModules() throws Exception {
    List<DatasetModuleId> toGrant = new ArrayList<>();
    // adding default modules to be available in dataset manager service
    for (Map.Entry<String, DatasetModule> module : defaultModules.entrySet()) {
        try {
            // NOTE: we assume default modules are always in classpath, hence passing null for jar location
            // NOTE: we add default modules in the system namespace
            DatasetModuleId defaultModule = NamespaceId.SYSTEM.datasetModule(module.getKey());
            typeManager.addModule(defaultModule, module.getValue().getClass().getName(), null, false);
            toGrant.add(defaultModule);
        } catch (DatasetModuleConflictException e) {
            // perfectly fine: we need to add default modules only the very first time service is started
            LOG.debug("Not adding {} module: it already exists", module.getKey());
        } catch (Throwable th) {
            LOG.error("Failed to add {} module. Aborting.", module.getKey(), th);
            throw Throwables.propagate(th);
        }
    }
    long startTime = System.currentTimeMillis();
    LOG.trace("Granting all privileges for {} default dataset modules. ", toGrant.size());
    for (DatasetModuleId defaultModule : toGrant) {
        grantAllPrivilegesOnModule(defaultModule, authenticationContext.getPrincipal());
    }
    long doneTime = System.currentTimeMillis();
    float elapsedSeconds = doneTime == startTime ? 0.0F : ((float) doneTime - startTime) / 1000;
    LOG.debug("Granting all privileges for {} default dataset modules took {} seconds.", toGrant.size(), elapsedSeconds);
}
Also used : DatasetModuleId(co.cask.cdap.proto.id.DatasetModuleId) DatasetModuleConflictException(co.cask.cdap.data2.datafabric.dataset.type.DatasetModuleConflictException) ArrayList(java.util.ArrayList) Map(java.util.Map) ImmutableMap(com.google.common.collect.ImmutableMap) LinkedHashMap(java.util.LinkedHashMap) DatasetModule(co.cask.cdap.api.dataset.module.DatasetModule)

Aggregations

DatasetModuleConflictException (co.cask.cdap.data2.datafabric.dataset.type.DatasetModuleConflictException)10 DatasetModuleMeta (co.cask.cdap.proto.DatasetModuleMeta)6 IOException (java.io.IOException)6 DatasetModuleCannotBeDeletedException (co.cask.cdap.common.DatasetModuleCannotBeDeletedException)5 DatasetModuleNotFoundException (co.cask.cdap.common.DatasetModuleNotFoundException)5 DatasetModuleId (co.cask.cdap.proto.id.DatasetModuleId)5 NamespaceId (co.cask.cdap.proto.id.NamespaceId)5 DatasetModule (co.cask.cdap.api.dataset.module.DatasetModule)4 ConflictException (co.cask.cdap.common.ConflictException)4 UnauthorizedException (co.cask.cdap.security.spi.authorization.UnauthorizedException)4 ImmutableMap (com.google.common.collect.ImmutableMap)4 LinkedHashMap (java.util.LinkedHashMap)4 Map (java.util.Map)4 Callable (java.util.concurrent.Callable)4 Location (org.apache.twill.filesystem.Location)4 DatasetTypeNotFoundException (co.cask.cdap.common.DatasetTypeNotFoundException)3 NamespaceNotFoundException (co.cask.cdap.common.NamespaceNotFoundException)3 NotFoundException (co.cask.cdap.common.NotFoundException)3 DatasetInstanceMDS (co.cask.cdap.data2.datafabric.dataset.service.mds.DatasetInstanceMDS)3 DatasetTypeMDS (co.cask.cdap.data2.datafabric.dataset.service.mds.DatasetTypeMDS)3