Examples with DatasetTypeId co.cask.cdap.proto.id.DatasetTypeId used on opensource projects

Example 11 with DatasetTypeId

use of co.cask.cdap.proto.id.DatasetTypeId in project cdap by caskdata.

the class DatasetClientTestRun method testSystemTypes.

@Test
public void testSystemTypes() throws Exception {
    // Tests that a dataset can be created in a namespace, even if the type does not exist in that namespace.
    // The dataset type is being resolved from the system namespace.
    DatasetTypeId type = TEST_NAMESPACE.datasetType(Table.class.getName());
    DatasetId instance = TEST_NAMESPACE.dataset("tableTypeDataset");
    Assert.assertFalse(typeClient.exists(type));
    Assert.assertFalse(datasetClient.exists(instance));
    datasetClient.create(instance, Table.class.getName());
    Assert.assertTrue(datasetClient.exists(instance));
}

Example 12 with DatasetTypeId

use of co.cask.cdap.proto.id.DatasetTypeId in project cdap by caskdata.

the class DatasetClientTestRun method testAll.

@Test
public void testAll() throws Exception {
    DatasetModuleId module = TEST_NAMESPACE.datasetModule(StandaloneDatasetModule.NAME);
    DatasetTypeId type = TEST_NAMESPACE.datasetType(StandaloneDataset.class.getName());
    DatasetModuleId moduleInOtherNamespace = OTHER_NAMESPACE.datasetModule(StandaloneDatasetModule.NAME);
    DatasetTypeId typeInOtherNamespace = OTHER_NAMESPACE.datasetType(StandaloneDataset.class.getName());
    int numBaseModules = moduleClient.list(TEST_NAMESPACE).size();
    int numBaseTypes = typeClient.list(TEST_NAMESPACE).size();
    LOG.info("Adding Dataset module");
    File moduleJarFile = createAppJarFile(StandaloneDatasetModule.class);
    moduleClient.add(TEST_NAMESPACE.datasetModule(StandaloneDatasetModule.NAME), StandaloneDatasetModule.class.getName(), moduleJarFile);
    Assert.assertEquals(numBaseModules + 1, moduleClient.list(TEST_NAMESPACE).size());
    Assert.assertEquals(numBaseTypes + 2, typeClient.list(TEST_NAMESPACE).size());
    LOG.info("Checking that the new Dataset module exists");
    DatasetModuleMeta datasetModuleMeta = moduleClient.get(module);
    Assert.assertNotNull(datasetModuleMeta);
    Assert.assertEquals(StandaloneDatasetModule.NAME, datasetModuleMeta.getName());
    LOG.info("Checking that the new Dataset module does not exist in a different namespace");
    try {
        moduleClient.get(moduleInOtherNamespace);
        Assert.fail("datasetModule found in namespace other than one in which it was expected");
    } catch (DatasetModuleNotFoundException expected) {
    // expected
    }
    LOG.info("Checking that the new Dataset type exists");
    DatasetTypeMeta datasetTypeMeta = typeClient.get(type);
    Assert.assertNotNull(datasetTypeMeta);
    Assert.assertEquals(type.getType(), datasetTypeMeta.getName());
    datasetTypeMeta = typeClient.get(type);
    Assert.assertNotNull(datasetTypeMeta);
    Assert.assertEquals(StandaloneDataset.class.getName(), datasetTypeMeta.getName());
    LOG.info("Checking that the new Dataset type does not exist in a different namespace");
    try {
        typeClient.get(typeInOtherNamespace);
        Assert.fail("datasetType found in namespace other than one in which it was expected");
    } catch (DatasetTypeNotFoundException expected) {
    // expected
    }
    LOG.info("Creating, truncating, and deleting dataset of new Dataset type");
    // Before creating dataset, there are some system datasets already exist
    int numBaseDataset = datasetClient.list(TEST_NAMESPACE).size();
    DatasetId instance = TEST_NAMESPACE.dataset("testDataset");
    String description = "test description";
    datasetClient.create(instance, new DatasetInstanceConfiguration(StandaloneDataset.TYPE_NAME, Collections.<String, String>emptyMap(), description, null));
    Assert.assertEquals(numBaseDataset + 1, datasetClient.list(TEST_NAMESPACE).size());
    // Assert dataset summary for the newly created dataset
    DatasetSpecificationSummary expectedSpec = new DatasetSpecificationSummary(instance.getDataset(), StandaloneDataset.TYPE_NAME, description, Collections.<String, String>emptyMap());
    Assert.assertEquals(expectedSpec, getSpecForDataset(instance, datasetClient.list(TEST_NAMESPACE)));
    datasetClient.truncate(instance);
    DatasetMeta metaBefore = datasetClient.get(instance);
    Assert.assertEquals(0, metaBefore.getSpec().getProperties().size());
    datasetClient.update(instance, ImmutableMap.of("sdf", "foo", "abc", "123"));
    DatasetMeta metaAfter = datasetClient.get(instance);
    Assert.assertEquals(2, metaAfter.getSpec().getProperties().size());
    Assert.assertTrue(metaAfter.getSpec().getProperties().containsKey("sdf"));
    Assert.assertTrue(metaAfter.getSpec().getProperties().containsKey("abc"));
    Assert.assertEquals("foo", metaAfter.getSpec().getProperties().get("sdf"));
    Assert.assertEquals("123", metaAfter.getSpec().getProperties().get("abc"));
    datasetClient.updateExisting(instance, ImmutableMap.of("sdf", "fzz"));
    metaAfter = datasetClient.get(instance);
    Assert.assertEquals(2, metaAfter.getSpec().getProperties().size());
    Assert.assertTrue(metaAfter.getSpec().getProperties().containsKey("sdf"));
    Assert.assertTrue(metaAfter.getSpec().getProperties().containsKey("abc"));
    Assert.assertEquals("fzz", metaAfter.getSpec().getProperties().get("sdf"));
    Assert.assertEquals("123", metaAfter.getSpec().getProperties().get("abc"));
    datasetClient.delete(instance);
    Assert.assertEquals(numBaseDataset, datasetClient.list(TEST_NAMESPACE).size());
    LOG.info("Creating and deleting multiple Datasets");
    for (int i = 1; i <= 3; i++) {
        datasetClient.create(TEST_NAMESPACE.dataset("testDataset" + i), StandaloneDataset.TYPE_NAME);
    }
    Assert.assertEquals(numBaseDataset + 3, datasetClient.list(TEST_NAMESPACE).size());
    for (int i = 1; i <= 3; i++) {
        datasetClient.delete(TEST_NAMESPACE.dataset("testDataset" + i));
    }
    Assert.assertEquals(numBaseDataset, datasetClient.list(TEST_NAMESPACE).size());
    LOG.info("Deleting Dataset module");
    moduleClient.delete(module);
    Assert.assertEquals(numBaseModules, moduleClient.list(TEST_NAMESPACE).size());
    Assert.assertEquals(numBaseTypes, typeClient.list(TEST_NAMESPACE).size());
    LOG.info("Adding Dataset module and then deleting all Dataset modules");
    moduleClient.add(TEST_NAMESPACE.datasetModule("testModule1"), StandaloneDatasetModule.class.getName(), moduleJarFile);
    Assert.assertEquals(numBaseModules + 1, moduleClient.list(TEST_NAMESPACE).size());
    Assert.assertEquals(numBaseTypes + 2, typeClient.list(TEST_NAMESPACE).size());
    moduleClient.deleteAll(TEST_NAMESPACE);
    Assert.assertEquals(numBaseModules, moduleClient.list(TEST_NAMESPACE).size());
    Assert.assertEquals(numBaseTypes, typeClient.list(TEST_NAMESPACE).size());
}

Example 13 with DatasetTypeId

use of co.cask.cdap.proto.id.DatasetTypeId in project cdap by caskdata.

the class DescribeDatasetTypeCommand method perform.

@Override
public void perform(Arguments arguments, PrintStream output) throws Exception {
    DatasetTypeId type = cliConfig.getCurrentNamespace().datasetType(arguments.get(ArgumentName.DATASET_TYPE.toString()));
    DatasetTypeMeta datasetTypeMeta = datasetTypeClient.get(type);
    Table table = Table.builder().setHeader("name", "modules").setRows(ImmutableList.of(datasetTypeMeta), new RowMaker<DatasetTypeMeta>() {

        @Override
        public List<?> makeRow(DatasetTypeMeta object) {
            return Lists.newArrayList(object.getName(), Joiner.on(", ").join(object.getModules()));
        }
    }).build();
    cliConfig.getTableRenderer().render(cliConfig, output, table);
}

Example 14 with DatasetTypeId

use of co.cask.cdap.proto.id.DatasetTypeId in project cdap by caskdata.

the class DatasetTypeManager method addModule.

/**
   * Add datasets module in a namespace
   *
   * @param datasetModuleId the {@link DatasetModuleId} to add
   * @param className module class
   * @param jarLocation location of the module jar
   * @param force if true, an update will be allowed even if there are conflicts with other modules, or if
   *                     removal of a type would break other modules' dependencies.
   */
public void addModule(final DatasetModuleId datasetModuleId, final String className, final Location jarLocation, final boolean force) throws DatasetModuleConflictException {
    LOG.debug("adding module: {}, className: {}, jarLocation: {}", datasetModuleId, className, jarLocation == null ? "[local]" : jarLocation);
    try {
        final DatasetTypeMDS datasetTypeMDS = datasetCache.getDataset(DatasetMetaTableUtil.META_TABLE_NAME);
        final DatasetInstanceMDS datasetInstanceMDS = datasetCache.getDataset(DatasetMetaTableUtil.INSTANCE_TABLE_NAME);
        txExecutorFactory.createExecutor(datasetCache).execute(new TransactionExecutor.Subroutine() {

            @Override
            public void apply() throws Exception {
                // 1. get existing module with all its types
                DatasetModuleMeta existing = datasetTypeMDS.getModule(datasetModuleId);
                DependencyTrackingRegistry reg;
                // 2. unpack jar and create class loader
                File unpackedLocation = Files.createTempDirectory(Files.createDirectories(systemTempPath), datasetModuleId.getEntityName()).toFile();
                DirectoryClassLoader cl = null;
                try {
                    // NOTE: if jarLocation is null, we assume that this is a system module, ie. always present in classpath
                    if (jarLocation != null) {
                        BundleJarUtil.unJar(jarLocation, unpackedLocation);
                        cl = new DirectoryClassLoader(cConf, unpackedLocation, FilterClassLoader.create(getClass().getClassLoader()), "lib");
                    }
                    reg = new DependencyTrackingRegistry(datasetModuleId, datasetTypeMDS, cl, force);
                    // 3. register the new module while tracking dependencies.
                    //    this will fail if a type exists in a different module
                    DatasetDefinitionRegistries.register(className, cl, reg);
                } catch (TypeConflictException e) {
                    // type conflict from the registry, we want to throw that as is
                    throw e;
                } catch (Exception e) {
                    LOG.error("Could not instantiate instance of dataset module class {} for module {} using jarLocation {}", className, datasetModuleId, jarLocation);
                    throw Throwables.propagate(e);
                } finally {
                    if (cl != null) {
                        // Close the ProgramClassLoader
                        Closeables.closeQuietly(cl);
                    }
                    try {
                        DirUtils.deleteDirectoryContents(unpackedLocation);
                    } catch (IOException e) {
                        LOG.warn("Failed to delete directory {}", unpackedLocation, e);
                    }
                }
                // 4. determine whether any type were removed from the module, and whether any other modules depend on them
                if (existing != null) {
                    Set<String> removedTypes = new HashSet<>(existing.getTypes());
                    removedTypes.removeAll(reg.getTypes());
                    // TODO (CDAP-6294): track dependencies at the type level
                    if (!force && !removedTypes.isEmpty() && !existing.getUsedByModules().isEmpty()) {
                        throw new DatasetModuleConflictException(String.format("Cannot update module '%s' to remove types %s: Modules %s may depend on it. Delete them first", datasetModuleId, removedTypes, existing.getUsedByModules()));
                    }
                    Collection<DatasetSpecification> instances = datasetInstanceMDS.getByTypes(datasetModuleId.getParent(), removedTypes);
                    if (!instances.isEmpty()) {
                        throw new DatasetModuleConflictException(String.format("Attempt to remove dataset types %s from module '%s' that have existing instances: %s. " + "Delete them first.", removedTypes, datasetModuleId, Iterables.toString(Iterables.transform(instances, new Function<DatasetSpecification, String>() {

                            @Nullable
                            @Override
                            public String apply(@Nullable DatasetSpecification input) {
                                return input.getName() + ":" + input.getType();
                            }
                        }))));
                    }
                }
                // NOTE: we use set to avoid duplicated dependencies
                // NOTE: we use LinkedHashSet to preserve order in which dependencies must be loaded
                Set<String> moduleDependencies = new LinkedHashSet<String>();
                for (DatasetTypeId usedType : reg.getUsedTypes()) {
                    DatasetModuleMeta usedModule = datasetTypeMDS.getModuleByType(usedType);
                    Preconditions.checkState(usedModule != null, String.format("Found a null used module for type %s for while adding module %s", usedType, datasetModuleId));
                    // adding all used types and the module itself, in this very order to keep the order of loading modules
                    // for instantiating a type
                    moduleDependencies.addAll(usedModule.getUsesModules());
                    boolean added = moduleDependencies.add(usedModule.getName());
                    if (added) {
                        // also adding this module as a dependent for all modules it uses
                        usedModule.addUsedByModule(datasetModuleId.getEntityName());
                        datasetTypeMDS.writeModule(usedType.getParent(), usedModule);
                    }
                }
                URI jarURI = jarLocation == null ? null : jarLocation.toURI();
                DatasetModuleMeta moduleMeta = existing == null ? new DatasetModuleMeta(datasetModuleId.getEntityName(), className, jarURI, reg.getTypes(), Lists.newArrayList(moduleDependencies)) : new DatasetModuleMeta(datasetModuleId.getEntityName(), className, jarURI, reg.getTypes(), Lists.newArrayList(moduleDependencies), Lists.newArrayList(existing.getUsedByModules()));
                datasetTypeMDS.writeModule(datasetModuleId.getParent(), moduleMeta);
            }
        });
    } catch (TransactionFailureException e) {
        Throwable cause = e.getCause();
        if (cause != null) {
            if (cause instanceof DatasetModuleConflictException) {
                throw (DatasetModuleConflictException) cause;
            } else if (cause instanceof TypeConflictException) {
                throw new DatasetModuleConflictException(cause.getMessage(), cause);
            }
        }
        throw Throwables.propagate(e);
    } catch (Exception e) {
        LOG.error("Operation failed", e);
        throw Throwables.propagate(e);
    }
}

Example 15 with DatasetTypeId

use of co.cask.cdap.proto.id.DatasetTypeId in project cdap by caskdata.

the class DatasetTypeService method grantAllPrivilegesOnModule.

private void grantAllPrivilegesOnModule(DatasetModuleId moduleId, Principal principal, @Nullable DatasetModuleMeta moduleMeta) throws Exception {
    privilegesManager.grant(moduleId, principal, EnumSet.allOf(Action.class));
    if (moduleMeta == null) {
        moduleMeta = typeManager.getModule(moduleId);
    }
    if (moduleMeta == null) {
        LOG.debug("Could not find metadata for module {}. Not granting privileges for its types.", moduleId);
        return;
    }
    for (String type : moduleMeta.getTypes()) {
        DatasetTypeId datasetTypeId = moduleId.getParent().datasetType(type);
        privilegesManager.grant(datasetTypeId, principal, EnumSet.allOf(Action.class));
    }
}