Examples with DatasetClassLoaderProvider io.cdap.cdap.data2.datafabric.dataset.type.DatasetClassLoaderProvider used on opensource projects

Example 1 with DatasetClassLoaderProvider

use of io.cdap.cdap.data2.datafabric.dataset.type.DatasetClassLoaderProvider in project cdap by caskdata.

the class DatasetAdminService method createOrUpdate.

/**
 * Configures and creates a Dataset
 *
 * @param datasetInstanceId dataset instance to be created
 * @param typeMeta type meta for the dataset
 * @param props dataset instance properties
 * @param existing if dataset already exists (in case of update), the existing properties
 * @return dataset specification
 */
public DatasetCreationResponse createOrUpdate(final DatasetId datasetInstanceId, final DatasetTypeMeta typeMeta, final DatasetProperties props, @Nullable final DatasetSpecification existing) throws Exception {
    if (existing == null) {
        LOG.info("Creating dataset instance {}, type meta: {}", datasetInstanceId, typeMeta);
    } else {
        LOG.info("Updating dataset instance {}, type meta: {}, existing: {}", datasetInstanceId, typeMeta, existing);
    }
    try (DatasetClassLoaderProvider classLoaderProvider = new DirectoryClassLoaderProvider(cConf, locationFactory)) {
        final DatasetContext context = DatasetContext.from(datasetInstanceId.getNamespace());
        UserGroupInformation ugi = getUgiForDataset(impersonator, datasetInstanceId);
        final DatasetType type = ImpersonationUtils.doAs(ugi, () -> {
            LOG.trace("Getting dataset type {}", typeMeta.getName());
            DatasetType type1 = dsFramework.getDatasetType(typeMeta, null, classLoaderProvider);
            if (type1 == null) {
                throw new BadRequestException(String.format("Cannot instantiate dataset type using provided type meta: %s", typeMeta));
            }
            LOG.trace("Got dataset type {}", typeMeta.getName());
            return type1;
        });
        DatasetSpecification spec = ImpersonationUtils.doAs(ugi, () -> {
            LOG.trace("Configuring dataset {} of type {}", datasetInstanceId.getDataset(), typeMeta.getName());
            DatasetSpecification spec1 = existing == null ? type.configure(datasetInstanceId.getEntityName(), props) : type.reconfigure(datasetInstanceId.getEntityName(), props, existing);
            LOG.trace("Configured dataset {} of type {}", datasetInstanceId.getDataset(), typeMeta.getName());
            DatasetAdmin admin = type.getAdmin(context, spec1);
            try {
                if (existing != null) {
                    if (admin instanceof Updatable) {
                        ((Updatable) admin).update(existing);
                    } else {
                        admin.upgrade();
                    }
                } else {
                    LOG.trace("Creating dataset {} of type {}", datasetInstanceId.getDataset(), typeMeta.getName());
                    admin.create();
                    LOG.trace("Created dataset {} of type {}", datasetInstanceId.getDataset(), typeMeta.getName());
                }
            } finally {
                Closeables.closeQuietly(admin);
            }
            return spec1;
        });
        // Writing system metadata should be done without impersonation since user may not have access to system tables.
        LOG.trace("Computing metadata for dataset {}", datasetInstanceId.getDataset());
        SystemMetadata metadata = computeSystemMetadata(datasetInstanceId, spec, props, typeMeta, type, context, existing != null, ugi);
        LOG.trace("Computed metadata for dataset {}", datasetInstanceId.getDataset());
        return new DatasetCreationResponse(spec, metadata);
    } catch (Exception e) {
        if (e instanceof IncompatibleUpdateException) {
            // this is expected to happen if user provides bad update properties, so we log this as debug
            LOG.debug("Incompatible update for dataset '{}'", datasetInstanceId, e);
        } else {
            LOG.error("Error {} dataset '{}': {}", existing == null ? "creating" : "updating", datasetInstanceId, e.getMessage(), e);
        }
        throw e;
    }
}

Example 2 with DatasetClassLoaderProvider

use of io.cdap.cdap.data2.datafabric.dataset.type.DatasetClassLoaderProvider in project cdap by caskdata.

the class DatasetAdminService method createOrUpdate.

/**
 * Configures and creates a Dataset
 *
 * @param datasetInstanceId dataset instance to be created
 * @param typeMeta type meta for the dataset
 * @param props dataset instance properties
 * @param existing if dataset already exists (in case of update), the existing properties
 * @return dataset specification
 */
public DatasetSpecification createOrUpdate(final DatasetId datasetInstanceId, final DatasetTypeMeta typeMeta, final DatasetProperties props, @Nullable final DatasetSpecification existing) throws Exception {
    if (existing == null) {
        LOG.info("Creating dataset instance {}, type meta: {}", datasetInstanceId, typeMeta);
    } else {
        LOG.info("Updating dataset instance {}, type meta: {}, existing: {}", datasetInstanceId, typeMeta, existing);
    }
    try (DatasetClassLoaderProvider classLoaderProvider = new DirectoryClassLoaderProvider(cConf, locationFactory)) {
        final DatasetContext context = DatasetContext.from(datasetInstanceId.getNamespace());
        UserGroupInformation ugi = getUgiForDataset(impersonator, datasetInstanceId);
        final DatasetType type = ImpersonationUtils.doAs(ugi, () -> {
            DatasetType type1 = dsFramework.getDatasetType(typeMeta, null, classLoaderProvider);
            if (type1 == null) {
                throw new BadRequestException(String.format("Cannot instantiate dataset type using provided type meta: %s", typeMeta));
            }
            return type1;
        });
        DatasetSpecification spec = ImpersonationUtils.doAs(ugi, () -> {
            DatasetSpecification spec1 = existing == null ? type.configure(datasetInstanceId.getEntityName(), props) : type.reconfigure(datasetInstanceId.getEntityName(), props, existing);
            DatasetAdmin admin = type.getAdmin(context, spec1);
            try {
                if (existing != null) {
                    if (admin instanceof Updatable) {
                        ((Updatable) admin).update(existing);
                    } else {
                        admin.upgrade();
                    }
                } else {
                    admin.create();
                }
            } finally {
                Closeables.closeQuietly(admin);
            }
            return spec1;
        });
        // Writing system metadata should be done without impersonation since user may not have access to system tables.
        writeSystemMetadata(datasetInstanceId, spec, props, typeMeta, type, context, existing != null, ugi);
        return spec;
    } catch (Exception e) {
        if (e instanceof IncompatibleUpdateException) {
            // this is expected to happen if user provides bad update properties, so we log this as debug
            LOG.debug("Incompatible update for dataset '{}'", datasetInstanceId, e);
        } else {
            LOG.error("Error {} dataset '{}': {}", existing == null ? "creating" : "updating", datasetInstanceId, e.getMessage(), e);
        }
        throw e;
    }
}

Example 3 with DatasetClassLoaderProvider

use of io.cdap.cdap.data2.datafabric.dataset.type.DatasetClassLoaderProvider in project cdap by caskdata.

the class DatasetAdminService method drop.

public void drop(final DatasetId datasetInstanceId, final DatasetTypeMeta typeMeta, final DatasetSpecification spec) throws Exception {
    LOG.info("Dropping dataset with spec: {}, type meta: {}", spec, typeMeta);
    try (DatasetClassLoaderProvider classLoaderProvider = new DirectoryClassLoaderProvider(cConf, locationFactory)) {
        UserGroupInformation ugi = getUgiForDataset(impersonator, datasetInstanceId);
        ImpersonationUtils.doAs(ugi, (Callable<Void>) () -> {
            DatasetType type = dsFramework.getDatasetType(typeMeta, null, classLoaderProvider);
            if (type == null) {
                throw new BadRequestException(String.format("Cannot instantiate dataset type using provided type meta: %s", typeMeta));
            }
            DatasetAdmin admin = type.getAdmin(DatasetContext.from(datasetInstanceId.getNamespace()), spec);
            try {
                admin.drop();
            } finally {
                Closeables.closeQuietly(admin);
            }
            return null;
        });
    }
    // Remove metadata for the dataset
    metadataStore.removeMetadata(datasetInstanceId);
}

Example 4 with DatasetClassLoaderProvider

use of io.cdap.cdap.data2.datafabric.dataset.type.DatasetClassLoaderProvider in project cdap by caskdata.

the class LineageWriterDatasetFramework method getDataset.

@Nullable
@Override
public <T extends Dataset> T getDataset(final DatasetId datasetInstanceId, final Map<String, String> arguments, @Nullable final ClassLoader classLoader, final DatasetClassLoaderProvider classLoaderProvider, @Nullable final Iterable<? extends EntityId> owners, final AccessType accessType) throws DatasetManagementException, IOException {
    Principal principal = authenticationContext.getPrincipal();
    try {
        // For system, skip authorization and lineage (user program shouldn't allow to access system dataset CDAP-6649)
        // For non-system dataset, always perform authorization and lineage.
        AccessEnforcer enforcer;
        DefaultDatasetRuntimeContext.DatasetAccessRecorder accessRecorder;
        if (!DatasetsUtil.isUserDataset(datasetInstanceId)) {
            enforcer = SYSTEM_NAMESPACE_ENFORCER;
            accessRecorder = SYSTEM_NAMESPACE_ACCESS_RECORDER;
        } else {
            enforcer = accessEnforcer;
            accessRecorder = new BasicDatasetAccessRecorder(datasetInstanceId, accessType, owners);
        }
        return DefaultDatasetRuntimeContext.execute(enforcer, accessRecorder, principal, datasetInstanceId, getConstructorDefaultAnnotation(accessType), () -> LineageWriterDatasetFramework.super.getDataset(datasetInstanceId, arguments, classLoader, classLoaderProvider, owners, accessType));
    } catch (IOException | DatasetManagementException | ServiceUnavailableException e) {
        throw e;
    } catch (Exception e) {
        throw new DatasetManagementException("Failed to create dataset instance: " + datasetInstanceId, e);
    }
}

Example 5 with DatasetClassLoaderProvider

use of io.cdap.cdap.data2.datafabric.dataset.type.DatasetClassLoaderProvider in project cdap by caskdata.

the class DatasetAdminService method drop.

public void drop(final DatasetId datasetInstanceId, final DatasetTypeMeta typeMeta, final DatasetSpecification spec) throws Exception {
    LOG.info("Dropping dataset with spec: {}, type meta: {}", spec, typeMeta);
    try (DatasetClassLoaderProvider classLoaderProvider = new DirectoryClassLoaderProvider(cConf, locationFactory)) {
        UserGroupInformation ugi = getUgiForDataset(impersonator, datasetInstanceId);
        ImpersonationUtils.doAs(ugi, (Callable<Void>) () -> {
            DatasetType type = dsFramework.getDatasetType(typeMeta, null, classLoaderProvider);
            if (type == null) {
                throw new BadRequestException(String.format("Cannot instantiate dataset type using provided type meta: %s", typeMeta));
            }
            DatasetAdmin admin = type.getAdmin(DatasetContext.from(datasetInstanceId.getNamespace()), spec);
            try {
                admin.drop();
            } finally {
                Closeables.closeQuietly(admin);
            }
            return null;
        });
    }
}