Examples with DatasetType io.cdap.cdap.data2.datafabric.dataset.DatasetType used on opensource projects

Example 6 with DatasetType

use of io.cdap.cdap.data2.datafabric.dataset.DatasetType in project cdap by cdapio.

the class DatasetAdminService method computeSystemMetadata.

private SystemMetadata computeSystemMetadata(DatasetId datasetInstanceId, final DatasetSpecification spec, DatasetProperties props, final DatasetTypeMeta typeMeta, final DatasetType type, final DatasetContext context, boolean existing, UserGroupInformation ugi) throws IOException {
    // add system metadata for user datasets only
    if (DatasetsUtil.isUserDataset(datasetInstanceId)) {
        Dataset dataset = null;
        try {
            try {
                dataset = ImpersonationUtils.doAs(ugi, () -> type.getDataset(context, spec, DatasetDefinition.NO_ARGUMENTS));
            } catch (Exception e) {
                LOG.warn("Exception while instantiating Dataset {}", datasetInstanceId, e);
            }
            // Make sure to write whatever system metadata that can be derived
            // even if the above instantiation throws exception
            DatasetSystemMetadataProvider metadataProvider;
            if (existing) {
                metadataProvider = new DatasetSystemMetadataProvider(datasetInstanceId, props, dataset, typeMeta.getName(), spec.getDescription());
            } else {
                long createTime = System.currentTimeMillis();
                metadataProvider = new DatasetSystemMetadataProvider(datasetInstanceId, props, createTime, dataset, typeMeta.getName(), spec.getDescription());
            }
            return new SystemMetadata(metadataProvider.getSystemPropertiesToAdd(), metadataProvider.getSystemTagsToAdd(), metadataProvider.getSchemaToAdd());
        } finally {
            if (dataset != null) {
                dataset.close();
            }
        }
    }
    return SystemMetadata.EMPTY;
}

Example 7 with DatasetType

use of io.cdap.cdap.data2.datafabric.dataset.DatasetType in project cdap by caskdata.

the class DatasetAdminService method createOrUpdate.

/**
 * Configures and creates a Dataset
 *
 * @param datasetInstanceId dataset instance to be created
 * @param typeMeta type meta for the dataset
 * @param props dataset instance properties
 * @param existing if dataset already exists (in case of update), the existing properties
 * @return dataset specification
 */
public DatasetSpecification createOrUpdate(final DatasetId datasetInstanceId, final DatasetTypeMeta typeMeta, final DatasetProperties props, @Nullable final DatasetSpecification existing) throws Exception {
    if (existing == null) {
        LOG.info("Creating dataset instance {}, type meta: {}", datasetInstanceId, typeMeta);
    } else {
        LOG.info("Updating dataset instance {}, type meta: {}, existing: {}", datasetInstanceId, typeMeta, existing);
    }
    try (DatasetClassLoaderProvider classLoaderProvider = new DirectoryClassLoaderProvider(cConf, locationFactory)) {
        final DatasetContext context = DatasetContext.from(datasetInstanceId.getNamespace());
        UserGroupInformation ugi = getUgiForDataset(impersonator, datasetInstanceId);
        final DatasetType type = ImpersonationUtils.doAs(ugi, () -> {
            DatasetType type1 = dsFramework.getDatasetType(typeMeta, null, classLoaderProvider);
            if (type1 == null) {
                throw new BadRequestException(String.format("Cannot instantiate dataset type using provided type meta: %s", typeMeta));
            }
            return type1;
        });
        DatasetSpecification spec = ImpersonationUtils.doAs(ugi, () -> {
            DatasetSpecification spec1 = existing == null ? type.configure(datasetInstanceId.getEntityName(), props) : type.reconfigure(datasetInstanceId.getEntityName(), props, existing);
            DatasetAdmin admin = type.getAdmin(context, spec1);
            try {
                if (existing != null) {
                    if (admin instanceof Updatable) {
                        ((Updatable) admin).update(existing);
                    } else {
                        admin.upgrade();
                    }
                } else {
                    admin.create();
                }
            } finally {
                Closeables.closeQuietly(admin);
            }
            return spec1;
        });
        // Writing system metadata should be done without impersonation since user may not have access to system tables.
        writeSystemMetadata(datasetInstanceId, spec, props, typeMeta, type, context, existing != null, ugi);
        return spec;
    } catch (Exception e) {
        if (e instanceof IncompatibleUpdateException) {
            // this is expected to happen if user provides bad update properties, so we log this as debug
            LOG.debug("Incompatible update for dataset '{}'", datasetInstanceId, e);
        } else {
            LOG.error("Error {} dataset '{}': {}", existing == null ? "creating" : "updating", datasetInstanceId, e.getMessage(), e);
        }
        throw e;
    }
}

Example 8 with DatasetType

use of io.cdap.cdap.data2.datafabric.dataset.DatasetType in project cdap by caskdata.

the class DatasetAdminService method drop.

public void drop(final DatasetId datasetInstanceId, final DatasetTypeMeta typeMeta, final DatasetSpecification spec) throws Exception {
    LOG.info("Dropping dataset with spec: {}, type meta: {}", spec, typeMeta);
    try (DatasetClassLoaderProvider classLoaderProvider = new DirectoryClassLoaderProvider(cConf, locationFactory)) {
        UserGroupInformation ugi = getUgiForDataset(impersonator, datasetInstanceId);
        ImpersonationUtils.doAs(ugi, (Callable<Void>) () -> {
            DatasetType type = dsFramework.getDatasetType(typeMeta, null, classLoaderProvider);
            if (type == null) {
                throw new BadRequestException(String.format("Cannot instantiate dataset type using provided type meta: %s", typeMeta));
            }
            DatasetAdmin admin = type.getAdmin(DatasetContext.from(datasetInstanceId.getNamespace()), spec);
            try {
                admin.drop();
            } finally {
                Closeables.closeQuietly(admin);
            }
            return null;
        });
    }
    // Remove metadata for the dataset
    metadataStore.removeMetadata(datasetInstanceId);
}

Example 9 with DatasetType

use of io.cdap.cdap.data2.datafabric.dataset.DatasetType in project cdap by caskdata.

the class InMemoryDatasetOpExecutor method drop.

@Override
public void drop(DatasetId datasetInstanceId, DatasetTypeMeta typeMeta, DatasetSpecification spec) throws Exception {
    DatasetType type = client.getDatasetType(typeMeta, null, new ConstantClassLoaderProvider());
    if (type == null) {
        throw new IllegalArgumentException("Dataset type cannot be instantiated for provided type meta: " + typeMeta);
    }
    DatasetAdmin admin = type.getAdmin(DatasetContext.from(datasetInstanceId.getNamespace()), spec);
    admin.drop();
}

Example 10 with DatasetType

use of io.cdap.cdap.data2.datafabric.dataset.DatasetType in project cdap by caskdata.

the class InMemoryDatasetOpExecutor method update.

@Override
public DatasetSpecification update(DatasetId datasetInstanceId, DatasetTypeMeta typeMeta, DatasetProperties props, DatasetSpecification existing) throws Exception {
    DatasetType type = client.getDatasetType(typeMeta, null, new ConstantClassLoaderProvider());
    if (type == null) {
        throw new IllegalArgumentException("Dataset type cannot be instantiated for provided type meta: " + typeMeta);
    }
    try {
        DatasetSpecification spec = type.reconfigure(datasetInstanceId.getEntityName(), props, existing);
        DatasetAdmin admin = type.getAdmin(DatasetContext.from(datasetInstanceId.getNamespace()), spec);
        if (admin instanceof Updatable) {
            ((Updatable) admin).update(existing);
        } else {
            admin.create();
        }
        if (spec.getDescription() == null && existing.getDescription() != null) {
            spec.setDescription(existing.getDescription());
        }
        return spec;
    } catch (IncompatibleUpdateException e) {
        throw new ConflictException(e.getMessage());
    }
}