Examples with DatasetModuleConflictException co.cask.cdap.data2.datafabric.dataset.type.DatasetModuleConflictException used on opensource projects

Example 11 with DatasetModuleConflictException

use of co.cask.cdap.data2.datafabric.dataset.type.DatasetModuleConflictException in project cdap by caskdata.

the class DefaultDatasetTypeService method createModuleConsumer.

private AbstractBodyConsumer createModuleConsumer(final DatasetModuleId datasetModuleId, final String className, final boolean forceUpdate) throws IOException, NotFoundException {
    final NamespaceId namespaceId = datasetModuleId.getParent();
    final Location namespaceHomeLocation;
    try {
        namespaceHomeLocation = impersonator.doAs(namespaceId, new Callable<Location>() {

            @Override
            public Location call() throws Exception {
                return namespacedLocationFactory.get(namespaceId);
            }
        });
    } catch (Exception e) {
        // the only checked exception that the callable throws is IOException
        Throwables.propagateIfInstanceOf(e, IOException.class);
        throw Throwables.propagate(e);
    }
    // verify namespace directory exists
    if (!namespaceHomeLocation.exists()) {
        String msg = String.format("Home directory %s for namespace %s not found", namespaceHomeLocation, namespaceId);
        LOG.debug(msg);
        throw new NotFoundException(msg);
    }
    // Store uploaded content to a local temp file
    String namespacesDir = cConf.get(Constants.Namespace.NAMESPACES_DIR);
    File localDataDir = new File(cConf.get(Constants.CFG_LOCAL_DATA_DIR));
    File namespaceBase = new File(localDataDir, namespacesDir);
    File tempDir = new File(new File(namespaceBase, datasetModuleId.getNamespace()), cConf.get(Constants.AppFabric.TEMP_DIR)).getAbsoluteFile();
    if (!DirUtils.mkdirs(tempDir)) {
        throw new IOException("Could not create temporary directory at: " + tempDir);
    }
    return new AbstractBodyConsumer(File.createTempFile("dataset-", ".jar", tempDir)) {

        @Override
        protected void onFinish(HttpResponder responder, File uploadedFile) throws Exception {
            if (className == null) {
                // We have to delay until body upload is completed due to the fact that not all client is
                // requesting with "Expect: 100-continue" header and the client library we have cannot handle
                // connection close, and yet be able to read response reliably.
                // In longer term we should fix the client, as well as the netty-http server. However, since
                // this handler will be gone in near future, it's ok to have this workaround.
                responder.sendString(HttpResponseStatus.BAD_REQUEST, "Required header 'class-name' is absent.");
                return;
            }
            LOG.debug("Adding module {}, class name: {}", datasetModuleId, className);
            String dataFabricDir = cConf.get(Constants.Dataset.Manager.OUTPUT_DIR);
            String moduleName = datasetModuleId.getModule();
            Location archiveDir = namespaceHomeLocation.append(dataFabricDir).append(moduleName).append(Constants.ARCHIVE_DIR);
            String archiveName = moduleName + ".jar";
            Location archive = archiveDir.append(archiveName);
            // Copy uploaded content to a temporary location
            Location tmpLocation = archive.getTempFile(".tmp");
            try {
                Locations.mkdirsIfNotExists(archiveDir);
                LOG.debug("Copy from {} to {}", uploadedFile, tmpLocation);
                Files.copy(uploadedFile, Locations.newOutputSupplier(tmpLocation));
                // Finally, move archive to final location
                LOG.debug("Storing module {} jar at {}", datasetModuleId, archive);
                if (tmpLocation.renameTo(archive) == null) {
                    throw new IOException(String.format("Could not move archive from location: %s, to location: %s", tmpLocation, archive));
                }
                typeManager.addModule(datasetModuleId, className, archive, forceUpdate);
                // todo: response with DatasetModuleMeta of just added module (and log this info)
                // Ideally this should have been done before, but we cannot grant privileges on types until they've been
                // added to the type MDS. First revoke any orphaned privileges for types left behind by past failed revokes
                LOG.info("Added module {}", datasetModuleId);
                responder.sendStatus(HttpResponseStatus.OK);
            } catch (Exception e) {
                // In case copy to temporary file failed, or rename failed
                try {
                    tmpLocation.delete();
                } catch (IOException ex) {
                    LOG.warn("Failed to cleanup temporary location {}", tmpLocation);
                }
                if (e instanceof DatasetModuleConflictException) {
                    responder.sendString(HttpResponseStatus.CONFLICT, e.getMessage());
                } else {
                    throw e;
                }
            }
        }
    };
}

Example 12 with DatasetModuleConflictException

use of co.cask.cdap.data2.datafabric.dataset.type.DatasetModuleConflictException in project cdap by caskdata.

the class DefaultDatasetTypeService method deployExtensionModules.

private void deployExtensionModules() {
    // adding any defined extension modules to be available in dataset manager service
    for (Map.Entry<String, DatasetModule> module : extensionModules.entrySet()) {
        try {
            // NOTE: we assume extension modules are always in classpath, hence passing null for jar location
            // NOTE: we add extension modules in the system namespace
            DatasetModuleId theModule = NamespaceId.SYSTEM.datasetModule(module.getKey());
            typeManager.addModule(theModule, module.getValue().getClass().getName(), null, false);
        } catch (DatasetModuleConflictException e) {
            // perfectly fine: we need to add the modules only the very first time service is started
            LOG.debug("Not adding {} extension module: it already exists", module.getKey());
        } catch (Throwable th) {
            LOG.error("Failed to add {} extension module. Aborting.", module.getKey(), th);
            throw Throwables.propagate(th);
        }
    }
}

Example 13 with DatasetModuleConflictException

use of co.cask.cdap.data2.datafabric.dataset.type.DatasetModuleConflictException in project cdap by caskdata.

the class DatasetTypeManager method addModule.

/**
 * Add datasets module in a namespace
 *
 * @param datasetModuleId the {@link DatasetModuleId} to add
 * @param className module class
 * @param jarLocation location of the module jar
 * @param force if true, an update will be allowed even if there are conflicts with other modules, or if
 *                     removal of a type would break other modules' dependencies.
 */
public void addModule(final DatasetModuleId datasetModuleId, final String className, final Location jarLocation, final boolean force) throws DatasetModuleConflictException {
    LOG.debug("adding module: {}, className: {}, jarLocation: {}", datasetModuleId, className, jarLocation == null ? "[local]" : jarLocation);
    try {
        final DatasetTypeMDS datasetTypeMDS = datasetCache.getDataset(DatasetMetaTableUtil.META_TABLE_NAME);
        final DatasetInstanceMDS datasetInstanceMDS = datasetCache.getDataset(DatasetMetaTableUtil.INSTANCE_TABLE_NAME);
        txExecutorFactory.createExecutor(datasetCache).execute(new TransactionExecutor.Subroutine() {

            @Override
            public void apply() throws Exception {
                // 1. get existing module with all its types
                DatasetModuleMeta existing = datasetTypeMDS.getModule(datasetModuleId);
                DependencyTrackingRegistry reg;
                // 2. unpack jar and create class loader
                File unpackedLocation = Files.createTempDirectory(Files.createDirectories(systemTempPath), datasetModuleId.getEntityName()).toFile();
                DirectoryClassLoader cl = null;
                try {
                    // NOTE: if jarLocation is null, we assume that this is a system module, ie. always present in classpath
                    if (jarLocation != null) {
                        BundleJarUtil.unJar(jarLocation, unpackedLocation);
                        cl = new DirectoryClassLoader(unpackedLocation, cConf.get(Constants.AppFabric.PROGRAM_EXTRA_CLASSPATH), FilterClassLoader.create(getClass().getClassLoader()), "lib");
                    }
                    reg = new DependencyTrackingRegistry(datasetModuleId, datasetTypeMDS, cl, force);
                    // 3. register the new module while tracking dependencies.
                    // this will fail if a type exists in a different module
                    DatasetDefinitionRegistries.register(className, cl, reg);
                } catch (TypeConflictException e) {
                    // type conflict from the registry, we want to throw that as is
                    throw e;
                } catch (Exception e) {
                    LOG.error("Could not instantiate instance of dataset module class {} for module {} using jarLocation {}", className, datasetModuleId, jarLocation);
                    throw Throwables.propagate(e);
                } finally {
                    if (cl != null) {
                        // Close the ProgramClassLoader
                        Closeables.closeQuietly(cl);
                    }
                    try {
                        DirUtils.deleteDirectoryContents(unpackedLocation);
                    } catch (IOException e) {
                        LOG.warn("Failed to delete directory {}", unpackedLocation, e);
                    }
                }
                // 4. determine whether any type were removed from the module, and whether any other modules depend on them
                if (existing != null) {
                    Set<String> removedTypes = new HashSet<>(existing.getTypes());
                    removedTypes.removeAll(reg.getTypes());
                    // TODO (CDAP-6294): track dependencies at the type level
                    if (!force && !removedTypes.isEmpty() && !existing.getUsedByModules().isEmpty()) {
                        throw new DatasetModuleConflictException(String.format("Cannot update module '%s' to remove types %s: Modules %s may depend on it. Delete them first", datasetModuleId, removedTypes, existing.getUsedByModules()));
                    }
                    Collection<DatasetSpecification> instances = datasetInstanceMDS.getByTypes(datasetModuleId.getParent(), removedTypes);
                    if (!instances.isEmpty()) {
                        throw new DatasetModuleConflictException(String.format("Attempt to remove dataset types %s from module '%s' that have existing instances: %s. " + "Delete them first.", removedTypes, datasetModuleId, Iterables.toString(Iterables.transform(instances, new Function<DatasetSpecification, String>() {

                            @Nullable
                            @Override
                            public String apply(@Nullable DatasetSpecification input) {
                                return input.getName() + ":" + input.getType();
                            }
                        }))));
                    }
                }
                // NOTE: we use set to avoid duplicated dependencies
                // NOTE: we use LinkedHashSet to preserve order in which dependencies must be loaded
                Set<String> moduleDependencies = new LinkedHashSet<String>();
                for (DatasetTypeId usedType : reg.getUsedTypes()) {
                    DatasetModuleMeta usedModule = datasetTypeMDS.getModuleByType(usedType);
                    Preconditions.checkState(usedModule != null, String.format("Found a null used module for type %s for while adding module %s", usedType, datasetModuleId));
                    // adding all used types and the module itself, in this very order to keep the order of loading modules
                    // for instantiating a type
                    moduleDependencies.addAll(usedModule.getUsesModules());
                    boolean added = moduleDependencies.add(usedModule.getName());
                    if (added) {
                        // also adding this module as a dependent for all modules it uses
                        usedModule.addUsedByModule(datasetModuleId.getEntityName());
                        datasetTypeMDS.writeModule(usedType.getParent(), usedModule);
                    }
                }
                URI jarURI = jarLocation == null ? null : jarLocation.toURI();
                DatasetModuleMeta moduleMeta = existing == null ? new DatasetModuleMeta(datasetModuleId.getEntityName(), className, jarURI, reg.getTypes(), Lists.newArrayList(moduleDependencies)) : new DatasetModuleMeta(datasetModuleId.getEntityName(), className, jarURI, reg.getTypes(), Lists.newArrayList(moduleDependencies), Lists.newArrayList(existing.getUsedByModules()));
                datasetTypeMDS.writeModule(datasetModuleId.getParent(), moduleMeta);
            }
        });
    } catch (TransactionFailureException e) {
        Throwable cause = e.getCause();
        if (cause != null) {
            if (cause instanceof DatasetModuleConflictException) {
                throw (DatasetModuleConflictException) cause;
            } else if (cause instanceof TypeConflictException) {
                throw new DatasetModuleConflictException(cause.getMessage(), cause);
            }
        }
        throw Throwables.propagate(e);
    } catch (Exception e) {
        LOG.error("Operation failed", e);
        throw Throwables.propagate(e);
    }
}