Examples with DatasetTypeMeta io.cdap.cdap.proto.DatasetTypeMeta used on opensource projects

Example 11 with DatasetTypeMeta

use of io.cdap.cdap.proto.DatasetTypeMeta in project cdap by caskdata.

the class DatasetInstanceService method update.

/**
 * Updates an existing Dataset specification properties.
 * {@link DatasetInstanceConfiguration} is constructed based on request and the Dataset instance is updated.
 *
 * @param instance the dataset instance
 * @param properties the dataset properties to be used
 * @throws NamespaceNotFoundException if the specified namespace was not found
 * @throws DatasetNotFoundException if the dataset was not found
 * @throws DatasetTypeNotFoundException if the type of the existing dataset was not found
 * @throws UnauthorizedException if perimeter security and authorization are enabled, and the current user does not
 *  have {@link StandardPermission#UPDATE} privilege on the #instance
 */
void update(DatasetId instance, Map<String, String> properties) throws Exception {
    ensureNamespaceExists(instance.getParent());
    Principal requestingUser = authenticationContext.getPrincipal();
    if (!DatasetsUtil.isSystemDatasetInUserNamespace(instance)) {
        accessEnforcer.enforce(instance, requestingUser, StandardPermission.UPDATE);
    }
    DatasetSpecification existing = instanceManager.get(instance);
    if (existing == null) {
        throw new DatasetNotFoundException(instance);
    }
    LOG.info("Update dataset {}, properties: {}", instance.getEntityName(), ConversionHelpers.toJson(properties));
    // by pass the auth check for dataset type when the operation is not creation
    DatasetTypeMeta typeMeta = getTypeInfo(instance.getParent(), existing.getType(), true);
    if (typeMeta == null) {
        // Type not found in the instance's namespace and the system namespace. Bail out.
        throw new DatasetTypeNotFoundException(ConversionHelpers.toDatasetTypeId(instance.getParent(), existing.getType()));
    }
    // Note how we execute configure() via opExecutorClient (outside of ds service) to isolate running user code
    DatasetProperties datasetProperties = DatasetProperties.of(properties);
    DatasetCreationResponse response = opExecutorClient.update(instance, typeMeta, datasetProperties, existing);
    DatasetSpecification spec = response.getSpec();
    instanceManager.add(instance.getParent(), spec);
    metaCache.invalidate(instance);
    updateExplore(instance, datasetProperties, existing, spec);
    publishAudit(instance, AuditType.UPDATE);
    publishMetadata(instance, response.getMetadata());
}

Example 12 with DatasetTypeMeta

use of io.cdap.cdap.proto.DatasetTypeMeta in project cdap by caskdata.

the class DatasetInstanceService method create.

/**
 * Creates a dataset instance.
 *
 * @param namespaceId the namespace to create the dataset instance in
 * @param name the name of the new dataset instance
 * @param props the properties for the new dataset instance
 * @throws NamespaceNotFoundException if the specified namespace was not found
 * @throws DatasetAlreadyExistsException if a dataset with the same name already exists
 * @throws DatasetTypeNotFoundException if the dataset type was not found
 * @throws UnauthorizedException if perimeter security and authorization are enabled, and the current user does not
 *  have {@link StandardPermission#UPDATE} privilege on the #instance's namespace
 */
void create(String namespaceId, String name, DatasetInstanceConfiguration props) throws Exception {
    NamespaceId namespace = ConversionHelpers.toNamespaceId(namespaceId);
    DatasetId datasetId = ConversionHelpers.toDatasetInstanceId(namespaceId, name);
    Principal requestingUser = authenticationContext.getPrincipal();
    String ownerPrincipal = props.getOwnerPrincipal();
    // need to enforce on the principal id if impersonation is involved
    KerberosPrincipalId effectiveOwner = SecurityUtil.getEffectiveOwner(ownerAdmin, namespace, ownerPrincipal);
    if (DatasetsUtil.isUserDataset(datasetId)) {
        LOG.trace("Authorizing impersonation for dataset {}", name);
        if (effectiveOwner != null) {
            accessEnforcer.enforce(effectiveOwner, requestingUser, AccessPermission.SET_OWNER);
        }
        accessEnforcer.enforce(datasetId, requestingUser, StandardPermission.CREATE);
        LOG.trace("Authorized impersonation for dataset {}", name);
    }
    LOG.trace("Ensuring existence of namespace {} for dataset {}", namespace, name);
    ensureNamespaceExists(namespace);
    LOG.trace("Ensured existence of namespace {} for dataset {}", namespace, name);
    LOG.trace("Retrieving instance metadata from MDS for dataset {}", name);
    DatasetSpecification existing = instanceManager.get(datasetId);
    if (existing != null) {
        throw new DatasetAlreadyExistsException(datasetId);
    }
    LOG.trace("Retrieved instance metadata from MDS for dataset {}", name);
    // for creation, we need enforcement for dataset type for user dataset, but bypass for system datasets
    DatasetTypeMeta typeMeta = getTypeInfo(namespace, props.getTypeName(), !DatasetsUtil.isUserDataset(datasetId));
    if (typeMeta == null) {
        // Type not found in the instance's namespace and the system namespace. Bail out.
        throw new DatasetTypeNotFoundException(ConversionHelpers.toDatasetTypeId(namespace, props.getTypeName()));
    }
    LOG.info("Creating dataset {}.{}, type name: {}, properties: {}", namespaceId, name, props.getTypeName(), props.getProperties());
    // exists or not
    if (ownerPrincipal != null) {
        LOG.trace("Adding owner for dataset {}", name);
        KerberosPrincipalId owner = new KerberosPrincipalId(ownerPrincipal);
        ownerAdmin.add(datasetId, owner);
        LOG.trace("Added owner {} for dataset {}", owner, name);
    }
    try {
        DatasetProperties datasetProperties = DatasetProperties.builder().addAll(props.getProperties()).setDescription(props.getDescription()).build();
        LOG.trace("Calling op executor service to configure dataset {}", name);
        DatasetCreationResponse response = opExecutorClient.create(datasetId, typeMeta, datasetProperties);
        LOG.trace("Received spec and metadata from op executor service for dataset {}: {}", name, response);
        LOG.trace("Adding instance metadata for dataset {}", name);
        DatasetSpecification spec = response.getSpec();
        instanceManager.add(namespace, spec);
        LOG.trace("Added instance metadata for dataset {}", name);
        metaCache.invalidate(datasetId);
        LOG.trace("Publishing audit for creation of dataset {}", name);
        publishAudit(datasetId, AuditType.CREATE);
        LOG.trace("Published audit for creation of dataset {}", name);
        SystemMetadata metadata = response.getMetadata();
        LOG.trace("Publishing system metadata for creation of dataset {}: {}", name, metadata);
        publishMetadata(datasetId, metadata);
        LOG.trace("Published system metadata for creation of dataset {}", name);
        // Enable explore
        enableExplore(datasetId, spec, props);
    } catch (Exception e) {
        // there was a problem in creating the dataset instance so delete the owner if it got added earlier
        // safe to call for entities which does not have an owner too
        ownerAdmin.delete(datasetId);
        throw e;
    }
}

Example 13 with DatasetTypeMeta

use of io.cdap.cdap.proto.DatasetTypeMeta in project cdap by caskdata.

the class DatasetInstanceService method getFromMds.

/**
 * Read the dataset meta data (instance and type) from MDS.
 *
 * Note this method cannot be called to create dataset instance, since it does not have enforcement on the dataset
 * type.
 */
private DatasetMeta getFromMds(DatasetId instance) throws Exception {
    // TODO: CDAP-3901 add back namespace existence check
    LOG.trace("Retrieving instance metadata from MDS for dataset {}", instance.getDataset());
    DatasetSpecification spec = instanceManager.get(instance);
    if (spec == null) {
        throw new NotFoundException(instance);
    }
    LOG.trace("Retrieved instance metadata from MDS for dataset {}", instance.getDataset());
    spec = DatasetsUtil.fixOriginalProperties(spec);
    DatasetTypeId datasetTypeId = instance.getParent().datasetType(spec.getType());
    // by pass the auth check for dataset type when the operation is not creation
    DatasetTypeMeta typeMeta = getTypeInfo(instance.getParent(), spec.getType(), true);
    if (typeMeta == null) {
        // TODO: This shouldn't happen unless CDAP is in an invalid state - maybe give different error
        throw new NotFoundException(datasetTypeId);
    }
    // for system dataset do not look up owner information in store as we know that it will be null.
    // Also, this is required for CDAP to start, because initially we don't want to look up owner admin
    // (causing its own lookup) as the SystemDatasetInitiator.getDataset is called when CDAP starts
    String ownerPrincipal = null;
    if (!NamespaceId.SYSTEM.equals(instance.getNamespaceId())) {
        LOG.trace("Retrieving owner principal for dataset {}", instance.getDataset());
        ownerPrincipal = ownerAdmin.getOwnerPrincipal(instance);
        LOG.trace("Retrieved owner principal for dataset {}", instance.getDataset());
    }
    return new DatasetMeta(spec, typeMeta, null, ownerPrincipal);
}

Example 14 with DatasetTypeMeta

use of io.cdap.cdap.proto.DatasetTypeMeta in project cdap by caskdata.

the class DatasetInstanceService method dropDataset.

/**
 * Drops a dataset.
 *
 * @param spec specification of dataset to be dropped.
 * @throws Exception on error.
 */
private void dropDataset(DatasetId instance, DatasetSpecification spec) throws Exception {
    LOG.info("Deleting dataset {}.{}", instance.getNamespace(), instance.getEntityName());
    disableExplore(instance, spec);
    if (!instanceManager.delete(instance)) {
        throw new DatasetNotFoundException(instance);
    }
    metaCache.invalidate(instance);
    // by pass the auth check for dataset type when the operation is not creation
    DatasetTypeMeta typeMeta = getTypeInfo(instance.getParent(), spec.getType(), true);
    if (typeMeta == null) {
        throw new DatasetNotFoundException(instance);
    }
    opExecutorClient.drop(instance, typeMeta, spec);
    // Remove metadata for the dataset
    LOG.trace("Removing metadata for dataset {}", instance);
    metadataServiceClient.drop(new MetadataMutation.Drop(instance.toMetadataEntity()));
    LOG.trace("Removed metadata for dataset {}", instance);
    publishAudit(instance, AuditType.DELETE);
    // deletes the owner principal for the entity if one was stored during creation
    ownerAdmin.delete(instance);
}

Example 15 with DatasetTypeMeta

use of io.cdap.cdap.proto.DatasetTypeMeta in project cdap by caskdata.

the class DatasetTypeTable method getTypeMeta.

private DatasetTypeMeta getTypeMeta(NamespaceId namespaceId, String typeName, DatasetModuleMeta moduleMeta) throws IOException {
    List<DatasetModuleMeta> modulesToLoad = Lists.newArrayList();
    // adding first all modules we depend on, then myself
    for (String usedModule : moduleMeta.getUsesModules()) {
        // Try to find module in the specified namespace first, then the system namespace
        DatasetModuleMeta usedModuleMeta = getModuleWithFallback(namespaceId.datasetModule(usedModule));
        // Module could not be found in either user or system namespace, bail out
        Preconditions.checkState(usedModuleMeta != null, String.format("Unable to find metadata about module %s that module %s uses.", usedModule, moduleMeta.getName()));
        modulesToLoad.add(usedModuleMeta);
    }
    modulesToLoad.add(moduleMeta);
    return new DatasetTypeMeta(typeName, modulesToLoad);
}