Search in sources :

Example 1 with DatasetTypeMeta

use of io.cdap.cdap.proto.DatasetTypeMeta in project cdap by caskdata.

the class ListDatasetTypesCommand method perform.

@Override
public void perform(Arguments arguments, PrintStream output) throws Exception {
    List<DatasetTypeMeta> datasetTypeMetas = datasetTypeClient.list(cliConfig.getCurrentNamespace());
    Table table = Table.builder().setHeader("name", "modules").setRows(datasetTypeMetas, new RowMaker<DatasetTypeMeta>() {

        @Override
        public List<?> makeRow(DatasetTypeMeta object) {
            List<String> modulesStrings = Lists.newArrayList();
            for (DatasetModuleMeta module : object.getModules()) {
                modulesStrings.add(module.getName());
            }
            return Lists.newArrayList(object.getName(), Joiner.on(", ").join(modulesStrings));
        }
    }).build();
    cliConfig.getTableRenderer().render(cliConfig, output, table);
}
Also used : Table(io.cdap.cdap.cli.util.table.Table) DatasetModuleMeta(io.cdap.cdap.proto.DatasetModuleMeta) RowMaker(io.cdap.cdap.cli.util.RowMaker) DatasetTypeMeta(io.cdap.cdap.proto.DatasetTypeMeta)

Example 2 with DatasetTypeMeta

use of io.cdap.cdap.proto.DatasetTypeMeta in project cdap by caskdata.

the class DatasetModulesDeployer method deployModules.

/**
 * Deploy the given dataset modules.
 *
 * @param namespaceId namespace to deploy to
 * @param modules the dataset modules to deploy
 * @param jarLocation the location of the jar file containing the modules
 * @param authorizingUser the authorizing user who will be making the call
 * @throws Exception if there was a problem deploying a module
 */
void deployModules(NamespaceId namespaceId, Map<String, String> modules, Location jarLocation, ClassLoader artifactClassLoader, String authorizingUser) throws Exception {
    List<String> implicitModules = new ArrayList<>();
    for (Map.Entry<String, String> moduleEntry : modules.entrySet()) {
        String moduleName = moduleEntry.getKey();
        String typeName = moduleEntry.getValue();
        if (systemDatasetFramework.hasSystemType(typeName)) {
            LOG.info("Not adding dataset type '{}' because it is defined by the system.", typeName);
            continue;
        }
        // Filter out the implicit modules: They have to be deployed last.
        if (moduleName.startsWith(".implicit.")) {
            implicitModules.add(typeName);
            continue;
        }
        loadAndDeployModule(artifactClassLoader, typeName, jarLocation, moduleName, namespaceId, authorizingUser);
    }
    for (String typeName : implicitModules) {
        final DatasetTypeId typeId = namespaceId.datasetType(typeName);
        DatasetTypeMeta typeMeta = AuthorizationUtil.authorizeAs(authorizingUser, new Callable<DatasetTypeMeta>() {

            @Override
            public DatasetTypeMeta call() throws Exception {
                return datasetFramework.getTypeInfo(typeId);
            }
        });
        if (typeMeta != null) {
            String existingModule = Iterables.getLast(typeMeta.getModules()).getName();
            if (modules.containsKey(existingModule)) {
                // it was deployed already as part of one of the explicit deployModule() calls
                continue;
            }
        }
        loadAndDeployModule(artifactClassLoader, typeName, jarLocation, typeName, namespaceId, authorizingUser);
    }
}
Also used : DatasetTypeId(io.cdap.cdap.proto.id.DatasetTypeId) ArrayList(java.util.ArrayList) DatasetTypeMeta(io.cdap.cdap.proto.DatasetTypeMeta) Map(java.util.Map) DatasetManagementException(io.cdap.cdap.api.dataset.DatasetManagementException) ModuleConflictException(io.cdap.cdap.data2.dataset2.ModuleConflictException)

Example 3 with DatasetTypeMeta

use of io.cdap.cdap.proto.DatasetTypeMeta in project cdap by caskdata.

the class DatasetTypeHandlerTest method verifyAll.

private void verifyAll(Set<DatasetModuleMeta> expectedModules, Map<String, List<String>> typeDependencies) throws IOException {
    Set<DatasetModuleMeta> actualModules = new HashSet<>(getModules().getResponseObject());
    Assert.assertEquals(expectedModules, actualModules);
    for (DatasetModuleMeta expectedModule : expectedModules) {
        ObjectResponse<DatasetModuleMeta> response = getModule(expectedModule.getName());
        Assert.assertEquals(HttpStatus.SC_OK, response.getResponseCode());
        Assert.assertEquals(expectedModule, response.getResponseObject());
        for (String type : expectedModule.getTypes()) {
            ObjectResponse<DatasetTypeMeta> typeResponse = getType(type);
            Assert.assertEquals(HttpStatus.SC_OK, typeResponse.getResponseCode());
            verify(typeResponse.getResponseObject(), type, typeDependencies.get(type));
        }
    }
    List<DatasetTypeMeta> actualTypes = getTypes().getResponseObject();
    Assert.assertEquals(actualTypes.size(), typeDependencies.size());
    Assert.assertTrue(Iterables.elementsEqual(typeDependencies.keySet(), actualTypes.stream().map(input -> input == null ? null : input.getName()).collect(Collectors.toList())));
    for (DatasetTypeMeta typeMeta : actualTypes) {
        verify(typeMeta, typeMeta.getName(), typeDependencies.get(typeMeta.getName()));
    }
}
Also used : Iterables(com.google.common.collect.Iterables) TypeToken(com.google.gson.reflect.TypeToken) BeforeClass(org.junit.BeforeClass) NamespaceId(io.cdap.cdap.proto.id.NamespaceId) Location(org.apache.twill.filesystem.Location) DatasetTypeMeta(io.cdap.cdap.proto.DatasetTypeMeta) DatasetInstanceConfiguration(io.cdap.cdap.proto.DatasetInstanceConfiguration) HttpResponse(io.cdap.common.http.HttpResponse) HttpStatus(org.apache.http.HttpStatus) HashMap(java.util.HashMap) ObjectResponse(io.cdap.common.http.ObjectResponse) HashSet(java.util.HashSet) ImmutableList(com.google.common.collect.ImmutableList) Map(java.util.Map) DatasetTypeId(io.cdap.cdap.proto.id.DatasetTypeId) HttpRequests(io.cdap.common.http.HttpRequests) DatasetModuleId(io.cdap.cdap.proto.id.DatasetModuleId) ImmutableSet(com.google.common.collect.ImmutableSet) ImmutableMap(com.google.common.collect.ImmutableMap) DatasetModuleMeta(io.cdap.cdap.proto.DatasetModuleMeta) Collection(java.util.Collection) Set(java.util.Set) Test(org.junit.Test) IOException(java.io.IOException) Collectors(java.util.stream.Collectors) Sets(com.google.common.collect.Sets) File(java.io.File) List(java.util.List) Assert(org.junit.Assert) Collections(java.util.Collections) HttpRequest(io.cdap.common.http.HttpRequest) DatasetModuleMeta(io.cdap.cdap.proto.DatasetModuleMeta) DatasetTypeMeta(io.cdap.cdap.proto.DatasetTypeMeta) HashSet(java.util.HashSet)

Example 4 with DatasetTypeMeta

use of io.cdap.cdap.proto.DatasetTypeMeta in project cdap by caskdata.

the class DatasetInstanceService method getTypeInfo.

/**
 * Finds the {@link DatasetTypeMeta} for the specified dataset type name.
 * Search order - first in the specified namespace, then in the 'system' namespace from defaultModules
 *
 * @param namespaceId {@link NamespaceId} for the specified namespace
 * @param typeName the name of the dataset type to search
 * @param byPassCheck a flag which determines whether to check privilege for the dataset type
 * @return {@link DatasetTypeMeta} for the type if found in either the specified namespace or in the system namespace,
 * null otherwise.
 * TODO: This may need to move to a util class eventually
 */
@Nullable
private DatasetTypeMeta getTypeInfo(NamespaceId namespaceId, String typeName, boolean byPassCheck) throws Exception {
    DatasetTypeId datasetTypeId = ConversionHelpers.toDatasetTypeId(namespaceId, typeName);
    try {
        LOG.trace("Retrieving metadata from mds for dataset type {} with authorization: {}", typeName, byPassCheck);
        DatasetTypeMeta meta = byPassCheck ? noAuthDatasetTypeService.getType(datasetTypeId) : authorizationDatasetTypeService.getType(datasetTypeId);
        LOG.trace("Retrieved metadata from mds for dataset type {}", typeName);
        return meta;
    } catch (DatasetTypeNotFoundException | UnauthorizedException e) {
        try {
            // Type not found in the instance's namespace. Now try finding it in the system namespace
            LOG.trace("Retrieving metadata from mds for system dataset type {}", typeName);
            DatasetTypeId systemDatasetTypeId = ConversionHelpers.toDatasetTypeId(NamespaceId.SYSTEM, typeName);
            LOG.trace("Retrieved metadata from mds for system dataset type {}", typeName);
            return noAuthDatasetTypeService.getType(systemDatasetTypeId);
        } catch (DatasetTypeNotFoundException exnWithSystemNS) {
            // if it's not found in system namespace, throw the original exception with the correct namespace
            throw e;
        }
    }
}
Also used : DatasetTypeId(io.cdap.cdap.proto.id.DatasetTypeId) DatasetTypeMeta(io.cdap.cdap.proto.DatasetTypeMeta) UnauthorizedException(io.cdap.cdap.security.spi.authorization.UnauthorizedException) DatasetTypeNotFoundException(io.cdap.cdap.common.DatasetTypeNotFoundException) Nullable(javax.annotation.Nullable)

Example 5 with DatasetTypeMeta

use of io.cdap.cdap.proto.DatasetTypeMeta in project cdap by caskdata.

the class DefaultDatasetTypeService method getType.

/**
 * Returns details of the specified {@link DatasetTypeId dataset type}.
 */
@Override
public DatasetTypeMeta getType(DatasetTypeId datasetTypeId) throws Exception {
    ensureNamespaceExists(datasetTypeId.getParent());
    DatasetTypeMeta typeMeta = typeManager.getTypeInfo(datasetTypeId);
    if (typeMeta == null) {
        throw new DatasetTypeNotFoundException(datasetTypeId);
    }
    return typeMeta;
}
Also used : DatasetTypeMeta(io.cdap.cdap.proto.DatasetTypeMeta) DatasetTypeNotFoundException(io.cdap.cdap.common.DatasetTypeNotFoundException)

Aggregations

DatasetTypeMeta (io.cdap.cdap.proto.DatasetTypeMeta)21 DatasetTypeNotFoundException (io.cdap.cdap.common.DatasetTypeNotFoundException)8 DatasetTypeId (io.cdap.cdap.proto.id.DatasetTypeId)7 DatasetSpecification (io.cdap.cdap.api.dataset.DatasetSpecification)5 DatasetModuleMeta (io.cdap.cdap.proto.DatasetModuleMeta)5 DatasetId (io.cdap.cdap.proto.id.DatasetId)5 DatasetProperties (io.cdap.cdap.api.dataset.DatasetProperties)4 DatasetNotFoundException (io.cdap.cdap.common.DatasetNotFoundException)4 DatasetModuleId (io.cdap.cdap.proto.id.DatasetModuleId)3 NamespaceId (io.cdap.cdap.proto.id.NamespaceId)3 HttpResponse (io.cdap.common.http.HttpResponse)3 RowMaker (io.cdap.cdap.cli.util.RowMaker)2 Table (io.cdap.cdap.cli.util.table.Table)2 DatasetTypeClient (io.cdap.cdap.client.DatasetTypeClient)2 NamespaceNotFoundException (io.cdap.cdap.common.NamespaceNotFoundException)2 NotFoundException (io.cdap.cdap.common.NotFoundException)2 DatasetCreationResponse (io.cdap.cdap.data2.datafabric.dataset.service.executor.DatasetCreationResponse)2 DatasetInstanceConfiguration (io.cdap.cdap.proto.DatasetInstanceConfiguration)2 DatasetMeta (io.cdap.cdap.proto.DatasetMeta)2 Principal (io.cdap.cdap.proto.security.Principal)2