Examples with DatasetCreationResponse io.cdap.cdap.data2.datafabric.dataset.service.executor.DatasetCreationResponse used on opensource projects

Example 1 with DatasetCreationResponse

use of io.cdap.cdap.data2.datafabric.dataset.service.executor.DatasetCreationResponse in project cdap by caskdata.

the class InMemoryDatasetOpExecutor method create.

@Override
public DatasetCreationResponse create(DatasetId datasetInstanceId, DatasetTypeMeta typeMeta, DatasetProperties props) throws Exception {
    DatasetType type = client.getDatasetType(typeMeta, null, new ConstantClassLoaderProvider());
    if (type == null) {
        throw new IllegalArgumentException("Dataset type cannot be instantiated for provided type meta: " + typeMeta);
    }
    DatasetSpecification spec = type.configure(datasetInstanceId.getEntityName(), props);
    DatasetAdmin admin = type.getAdmin(DatasetContext.from(datasetInstanceId.getNamespace()), spec);
    admin.create();
    return new DatasetCreationResponse(spec, null);
}

Example 2 with DatasetCreationResponse

use of io.cdap.cdap.data2.datafabric.dataset.service.executor.DatasetCreationResponse in project cdap by caskdata.

the class InMemoryDatasetOpExecutor method update.

@Override
public DatasetCreationResponse update(DatasetId datasetInstanceId, DatasetTypeMeta typeMeta, DatasetProperties props, DatasetSpecification existing) throws Exception {
    DatasetType type = client.getDatasetType(typeMeta, null, new ConstantClassLoaderProvider());
    if (type == null) {
        throw new IllegalArgumentException("Dataset type cannot be instantiated for provided type meta: " + typeMeta);
    }
    try {
        DatasetSpecification spec = type.reconfigure(datasetInstanceId.getEntityName(), props, existing);
        DatasetAdmin admin = type.getAdmin(DatasetContext.from(datasetInstanceId.getNamespace()), spec);
        if (admin instanceof Updatable) {
            ((Updatable) admin).update(existing);
        } else {
            admin.create();
        }
        if (spec.getDescription() == null && existing.getDescription() != null) {
            spec.setDescription(existing.getDescription());
        }
        return new DatasetCreationResponse(spec, null);
    } catch (IncompatibleUpdateException e) {
        throw new ConflictException(e.getMessage());
    }
}

Example 3 with DatasetCreationResponse

use of io.cdap.cdap.data2.datafabric.dataset.service.executor.DatasetCreationResponse in project cdap by caskdata.

the class DatasetAdminService method createOrUpdate.

/**
 * Configures and creates a Dataset
 *
 * @param datasetInstanceId dataset instance to be created
 * @param typeMeta type meta for the dataset
 * @param props dataset instance properties
 * @param existing if dataset already exists (in case of update), the existing properties
 * @return dataset specification
 */
public DatasetCreationResponse createOrUpdate(final DatasetId datasetInstanceId, final DatasetTypeMeta typeMeta, final DatasetProperties props, @Nullable final DatasetSpecification existing) throws Exception {
    if (existing == null) {
        LOG.info("Creating dataset instance {}, type meta: {}", datasetInstanceId, typeMeta);
    } else {
        LOG.info("Updating dataset instance {}, type meta: {}, existing: {}", datasetInstanceId, typeMeta, existing);
    }
    try (DatasetClassLoaderProvider classLoaderProvider = new DirectoryClassLoaderProvider(cConf, locationFactory)) {
        final DatasetContext context = DatasetContext.from(datasetInstanceId.getNamespace());
        UserGroupInformation ugi = getUgiForDataset(impersonator, datasetInstanceId);
        final DatasetType type = ImpersonationUtils.doAs(ugi, () -> {
            LOG.trace("Getting dataset type {}", typeMeta.getName());
            DatasetType type1 = dsFramework.getDatasetType(typeMeta, null, classLoaderProvider);
            if (type1 == null) {
                throw new BadRequestException(String.format("Cannot instantiate dataset type using provided type meta: %s", typeMeta));
            }
            LOG.trace("Got dataset type {}", typeMeta.getName());
            return type1;
        });
        DatasetSpecification spec = ImpersonationUtils.doAs(ugi, () -> {
            LOG.trace("Configuring dataset {} of type {}", datasetInstanceId.getDataset(), typeMeta.getName());
            DatasetSpecification spec1 = existing == null ? type.configure(datasetInstanceId.getEntityName(), props) : type.reconfigure(datasetInstanceId.getEntityName(), props, existing);
            LOG.trace("Configured dataset {} of type {}", datasetInstanceId.getDataset(), typeMeta.getName());
            DatasetAdmin admin = type.getAdmin(context, spec1);
            try {
                if (existing != null) {
                    if (admin instanceof Updatable) {
                        ((Updatable) admin).update(existing);
                    } else {
                        admin.upgrade();
                    }
                } else {
                    LOG.trace("Creating dataset {} of type {}", datasetInstanceId.getDataset(), typeMeta.getName());
                    admin.create();
                    LOG.trace("Created dataset {} of type {}", datasetInstanceId.getDataset(), typeMeta.getName());
                }
            } finally {
                Closeables.closeQuietly(admin);
            }
            return spec1;
        });
        // Writing system metadata should be done without impersonation since user may not have access to system tables.
        LOG.trace("Computing metadata for dataset {}", datasetInstanceId.getDataset());
        SystemMetadata metadata = computeSystemMetadata(datasetInstanceId, spec, props, typeMeta, type, context, existing != null, ugi);
        LOG.trace("Computed metadata for dataset {}", datasetInstanceId.getDataset());
        return new DatasetCreationResponse(spec, metadata);
    } catch (Exception e) {
        if (e instanceof IncompatibleUpdateException) {
            // this is expected to happen if user provides bad update properties, so we log this as debug
            LOG.debug("Incompatible update for dataset '{}'", datasetInstanceId, e);
        } else {
            LOG.error("Error {} dataset '{}': {}", existing == null ? "creating" : "updating", datasetInstanceId, e.getMessage(), e);
        }
        throw e;
    }
}

Example 4 with DatasetCreationResponse

use of io.cdap.cdap.data2.datafabric.dataset.service.executor.DatasetCreationResponse in project cdap by caskdata.

the class DatasetInstanceService method update.

/**
 * Updates an existing Dataset specification properties.
 * {@link DatasetInstanceConfiguration} is constructed based on request and the Dataset instance is updated.
 *
 * @param instance the dataset instance
 * @param properties the dataset properties to be used
 * @throws NamespaceNotFoundException if the specified namespace was not found
 * @throws DatasetNotFoundException if the dataset was not found
 * @throws DatasetTypeNotFoundException if the type of the existing dataset was not found
 * @throws UnauthorizedException if perimeter security and authorization are enabled, and the current user does not
 *  have {@link StandardPermission#UPDATE} privilege on the #instance
 */
void update(DatasetId instance, Map<String, String> properties) throws Exception {
    ensureNamespaceExists(instance.getParent());
    Principal requestingUser = authenticationContext.getPrincipal();
    if (!DatasetsUtil.isSystemDatasetInUserNamespace(instance)) {
        accessEnforcer.enforce(instance, requestingUser, StandardPermission.UPDATE);
    }
    DatasetSpecification existing = instanceManager.get(instance);
    if (existing == null) {
        throw new DatasetNotFoundException(instance);
    }
    LOG.info("Update dataset {}, properties: {}", instance.getEntityName(), ConversionHelpers.toJson(properties));
    // by pass the auth check for dataset type when the operation is not creation
    DatasetTypeMeta typeMeta = getTypeInfo(instance.getParent(), existing.getType(), true);
    if (typeMeta == null) {
        // Type not found in the instance's namespace and the system namespace. Bail out.
        throw new DatasetTypeNotFoundException(ConversionHelpers.toDatasetTypeId(instance.getParent(), existing.getType()));
    }
    // Note how we execute configure() via opExecutorClient (outside of ds service) to isolate running user code
    DatasetProperties datasetProperties = DatasetProperties.of(properties);
    DatasetCreationResponse response = opExecutorClient.update(instance, typeMeta, datasetProperties, existing);
    DatasetSpecification spec = response.getSpec();
    instanceManager.add(instance.getParent(), spec);
    metaCache.invalidate(instance);
    updateExplore(instance, datasetProperties, existing, spec);
    publishAudit(instance, AuditType.UPDATE);
    publishMetadata(instance, response.getMetadata());
}

Example 5 with DatasetCreationResponse

use of io.cdap.cdap.data2.datafabric.dataset.service.executor.DatasetCreationResponse in project cdap by caskdata.

the class DatasetInstanceService method create.

/**
 * Creates a dataset instance.
 *
 * @param namespaceId the namespace to create the dataset instance in
 * @param name the name of the new dataset instance
 * @param props the properties for the new dataset instance
 * @throws NamespaceNotFoundException if the specified namespace was not found
 * @throws DatasetAlreadyExistsException if a dataset with the same name already exists
 * @throws DatasetTypeNotFoundException if the dataset type was not found
 * @throws UnauthorizedException if perimeter security and authorization are enabled, and the current user does not
 *  have {@link StandardPermission#UPDATE} privilege on the #instance's namespace
 */
void create(String namespaceId, String name, DatasetInstanceConfiguration props) throws Exception {
    NamespaceId namespace = ConversionHelpers.toNamespaceId(namespaceId);
    DatasetId datasetId = ConversionHelpers.toDatasetInstanceId(namespaceId, name);
    Principal requestingUser = authenticationContext.getPrincipal();
    String ownerPrincipal = props.getOwnerPrincipal();
    // need to enforce on the principal id if impersonation is involved
    KerberosPrincipalId effectiveOwner = SecurityUtil.getEffectiveOwner(ownerAdmin, namespace, ownerPrincipal);
    if (DatasetsUtil.isUserDataset(datasetId)) {
        LOG.trace("Authorizing impersonation for dataset {}", name);
        if (effectiveOwner != null) {
            accessEnforcer.enforce(effectiveOwner, requestingUser, AccessPermission.SET_OWNER);
        }
        accessEnforcer.enforce(datasetId, requestingUser, StandardPermission.CREATE);
        LOG.trace("Authorized impersonation for dataset {}", name);
    }
    LOG.trace("Ensuring existence of namespace {} for dataset {}", namespace, name);
    ensureNamespaceExists(namespace);
    LOG.trace("Ensured existence of namespace {} for dataset {}", namespace, name);
    LOG.trace("Retrieving instance metadata from MDS for dataset {}", name);
    DatasetSpecification existing = instanceManager.get(datasetId);
    if (existing != null) {
        throw new DatasetAlreadyExistsException(datasetId);
    }
    LOG.trace("Retrieved instance metadata from MDS for dataset {}", name);
    // for creation, we need enforcement for dataset type for user dataset, but bypass for system datasets
    DatasetTypeMeta typeMeta = getTypeInfo(namespace, props.getTypeName(), !DatasetsUtil.isUserDataset(datasetId));
    if (typeMeta == null) {
        // Type not found in the instance's namespace and the system namespace. Bail out.
        throw new DatasetTypeNotFoundException(ConversionHelpers.toDatasetTypeId(namespace, props.getTypeName()));
    }
    LOG.info("Creating dataset {}.{}, type name: {}, properties: {}", namespaceId, name, props.getTypeName(), props.getProperties());
    // exists or not
    if (ownerPrincipal != null) {
        LOG.trace("Adding owner for dataset {}", name);
        KerberosPrincipalId owner = new KerberosPrincipalId(ownerPrincipal);
        ownerAdmin.add(datasetId, owner);
        LOG.trace("Added owner {} for dataset {}", owner, name);
    }
    try {
        DatasetProperties datasetProperties = DatasetProperties.builder().addAll(props.getProperties()).setDescription(props.getDescription()).build();
        LOG.trace("Calling op executor service to configure dataset {}", name);
        DatasetCreationResponse response = opExecutorClient.create(datasetId, typeMeta, datasetProperties);
        LOG.trace("Received spec and metadata from op executor service for dataset {}: {}", name, response);
        LOG.trace("Adding instance metadata for dataset {}", name);
        DatasetSpecification spec = response.getSpec();
        instanceManager.add(namespace, spec);
        LOG.trace("Added instance metadata for dataset {}", name);
        metaCache.invalidate(datasetId);
        LOG.trace("Publishing audit for creation of dataset {}", name);
        publishAudit(datasetId, AuditType.CREATE);
        LOG.trace("Published audit for creation of dataset {}", name);
        SystemMetadata metadata = response.getMetadata();
        LOG.trace("Publishing system metadata for creation of dataset {}: {}", name, metadata);
        publishMetadata(datasetId, metadata);
        LOG.trace("Published system metadata for creation of dataset {}", name);
        // Enable explore
        enableExplore(datasetId, spec, props);
    } catch (Exception e) {
        // there was a problem in creating the dataset instance so delete the owner if it got added earlier
        // safe to call for entities which does not have an owner too
        ownerAdmin.delete(datasetId);
        throw e;
    }
}