Examples with SystemMetadata io.cdap.cdap.data2.metadata.system.SystemMetadata used on opensource projects

Example 6 with SystemMetadata

use of io.cdap.cdap.data2.metadata.system.SystemMetadata in project cdap by caskdata.

the class DatasetInstanceService method publishMetadata.

private void publishMetadata(DatasetId dataset, SystemMetadata metadata) {
    if (metadata != null && !metadata.isEmpty()) {
        SystemMetadataWriter metadataWriter = new DelegateSystemMetadataWriter(metadataServiceClient, dataset, metadata);
        metadataWriter.write();
    }
}

Example 7 with SystemMetadata

use of io.cdap.cdap.data2.metadata.system.SystemMetadata in project cdap by caskdata.

the class DatasetInstanceService method create.

/**
 * Creates a dataset instance.
 *
 * @param namespaceId the namespace to create the dataset instance in
 * @param name the name of the new dataset instance
 * @param props the properties for the new dataset instance
 * @throws NamespaceNotFoundException if the specified namespace was not found
 * @throws DatasetAlreadyExistsException if a dataset with the same name already exists
 * @throws DatasetTypeNotFoundException if the dataset type was not found
 * @throws UnauthorizedException if perimeter security and authorization are enabled, and the current user does not
 *  have {@link StandardPermission#UPDATE} privilege on the #instance's namespace
 */
void create(String namespaceId, String name, DatasetInstanceConfiguration props) throws Exception {
    NamespaceId namespace = ConversionHelpers.toNamespaceId(namespaceId);
    DatasetId datasetId = ConversionHelpers.toDatasetInstanceId(namespaceId, name);
    Principal requestingUser = authenticationContext.getPrincipal();
    String ownerPrincipal = props.getOwnerPrincipal();
    // need to enforce on the principal id if impersonation is involved
    KerberosPrincipalId effectiveOwner = SecurityUtil.getEffectiveOwner(ownerAdmin, namespace, ownerPrincipal);
    if (DatasetsUtil.isUserDataset(datasetId)) {
        LOG.trace("Authorizing impersonation for dataset {}", name);
        if (effectiveOwner != null) {
            accessEnforcer.enforce(effectiveOwner, requestingUser, AccessPermission.SET_OWNER);
        }
        accessEnforcer.enforce(datasetId, requestingUser, StandardPermission.CREATE);
        LOG.trace("Authorized impersonation for dataset {}", name);
    }
    LOG.trace("Ensuring existence of namespace {} for dataset {}", namespace, name);
    ensureNamespaceExists(namespace);
    LOG.trace("Ensured existence of namespace {} for dataset {}", namespace, name);
    LOG.trace("Retrieving instance metadata from MDS for dataset {}", name);
    DatasetSpecification existing = instanceManager.get(datasetId);
    if (existing != null) {
        throw new DatasetAlreadyExistsException(datasetId);
    }
    LOG.trace("Retrieved instance metadata from MDS for dataset {}", name);
    // for creation, we need enforcement for dataset type for user dataset, but bypass for system datasets
    DatasetTypeMeta typeMeta = getTypeInfo(namespace, props.getTypeName(), !DatasetsUtil.isUserDataset(datasetId));
    if (typeMeta == null) {
        // Type not found in the instance's namespace and the system namespace. Bail out.
        throw new DatasetTypeNotFoundException(ConversionHelpers.toDatasetTypeId(namespace, props.getTypeName()));
    }
    LOG.info("Creating dataset {}.{}, type name: {}, properties: {}", namespaceId, name, props.getTypeName(), props.getProperties());
    // exists or not
    if (ownerPrincipal != null) {
        LOG.trace("Adding owner for dataset {}", name);
        KerberosPrincipalId owner = new KerberosPrincipalId(ownerPrincipal);
        ownerAdmin.add(datasetId, owner);
        LOG.trace("Added owner {} for dataset {}", owner, name);
    }
    try {
        DatasetProperties datasetProperties = DatasetProperties.builder().addAll(props.getProperties()).setDescription(props.getDescription()).build();
        LOG.trace("Calling op executor service to configure dataset {}", name);
        DatasetCreationResponse response = opExecutorClient.create(datasetId, typeMeta, datasetProperties);
        LOG.trace("Received spec and metadata from op executor service for dataset {}: {}", name, response);
        LOG.trace("Adding instance metadata for dataset {}", name);
        DatasetSpecification spec = response.getSpec();
        instanceManager.add(namespace, spec);
        LOG.trace("Added instance metadata for dataset {}", name);
        metaCache.invalidate(datasetId);
        LOG.trace("Publishing audit for creation of dataset {}", name);
        publishAudit(datasetId, AuditType.CREATE);
        LOG.trace("Published audit for creation of dataset {}", name);
        SystemMetadata metadata = response.getMetadata();
        LOG.trace("Publishing system metadata for creation of dataset {}: {}", name, metadata);
        publishMetadata(datasetId, metadata);
        LOG.trace("Published system metadata for creation of dataset {}", name);
        // Enable explore
        enableExplore(datasetId, spec, props);
    } catch (Exception e) {
        // there was a problem in creating the dataset instance so delete the owner if it got added earlier
        // safe to call for entities which does not have an owner too
        ownerAdmin.delete(datasetId);
        throw e;
    }
}

Example 8 with SystemMetadata

use of io.cdap.cdap.data2.metadata.system.SystemMetadata in project cdap by caskdata.

the class DatasetAdminService method computeSystemMetadata.

private SystemMetadata computeSystemMetadata(DatasetId datasetInstanceId, final DatasetSpecification spec, DatasetProperties props, final DatasetTypeMeta typeMeta, final DatasetType type, final DatasetContext context, boolean existing, UserGroupInformation ugi) throws IOException {
    // add system metadata for user datasets only
    if (DatasetsUtil.isUserDataset(datasetInstanceId)) {
        Dataset dataset = null;
        try {
            try {
                dataset = ImpersonationUtils.doAs(ugi, () -> type.getDataset(context, spec, DatasetDefinition.NO_ARGUMENTS));
            } catch (Exception e) {
                LOG.warn("Exception while instantiating Dataset {}", datasetInstanceId, e);
            }
            // Make sure to write whatever system metadata that can be derived
            // even if the above instantiation throws exception
            DatasetSystemMetadataProvider metadataProvider;
            if (existing) {
                metadataProvider = new DatasetSystemMetadataProvider(datasetInstanceId, props, dataset, typeMeta.getName(), spec.getDescription());
            } else {
                long createTime = System.currentTimeMillis();
                metadataProvider = new DatasetSystemMetadataProvider(datasetInstanceId, props, createTime, dataset, typeMeta.getName(), spec.getDescription());
            }
            return new SystemMetadata(metadataProvider.getSystemPropertiesToAdd(), metadataProvider.getSystemTagsToAdd(), metadataProvider.getSchemaToAdd());
        } finally {
            if (dataset != null) {
                dataset.close();
            }
        }
    }
    return SystemMetadata.EMPTY;
}

Example 9 with SystemMetadata

use of io.cdap.cdap.data2.metadata.system.SystemMetadata in project cdap by caskdata.

the class SearchHelper method addMetadataToEntities.

private Set<MetadataSearchResultRecord> addMetadataToEntities(Set<MetadataEntity> entities, Map<MetadataEntity, MetadataDataset.Record> systemMetadata, Map<MetadataEntity, MetadataDataset.Record> userMetadata) {
    Set<MetadataSearchResultRecord> result = new LinkedHashSet<>();
    for (MetadataEntity entity : entities) {
        ImmutableMap.Builder<MetadataScope, Metadata> builder = ImmutableMap.builder();
        // Add system metadata
        MetadataDataset.Record metadata = systemMetadata.get(entity);
        if (metadata != null) {
            builder.put(SYSTEM, new Metadata(metadata.getProperties(), metadata.getTags()));
        }
        // Add user metadata
        metadata = userMetadata.get(entity);
        if (metadata != null) {
            builder.put(MetadataScope.USER, new Metadata(metadata.getProperties(), metadata.getTags()));
        }
        // Create result
        result.add(new MetadataSearchResultRecord(entity, builder.build()));
    }
    return result;
}

Example 10 with SystemMetadata

use of io.cdap.cdap.data2.metadata.system.SystemMetadata in project cdap by cdapio.

the class DatasetInstanceService method publishMetadata.

private void publishMetadata(DatasetId dataset, SystemMetadata metadata) {
    if (metadata != null && !metadata.isEmpty()) {
        SystemMetadataWriter metadataWriter = new DelegateSystemMetadataWriter(metadataServiceClient, dataset, metadata);
        metadataWriter.write();
    }
}