Examples with DatasetFramework co.cask.cdap.data2.dataset2.DatasetFramework used on opensource projects

Example 6 with DatasetFramework

use of co.cask.cdap.data2.dataset2.DatasetFramework in project cdap by caskdata.

the class ExistingEntitySystemMetadataWriter method writeSystemMetadataForDatasets.

private void writeSystemMetadataForDatasets(NamespaceId namespace, DatasetFramework dsFramework) throws DatasetManagementException, IOException, NamespaceNotFoundException {
    SystemDatasetInstantiatorFactory systemDatasetInstantiatorFactory = new SystemDatasetInstantiatorFactory(locationFactory, dsFramework, cConf);
    try (SystemDatasetInstantiator systemDatasetInstantiator = systemDatasetInstantiatorFactory.create()) {
        for (DatasetSpecificationSummary summary : dsFramework.getInstances(namespace)) {
            final DatasetId dsInstance = namespace.dataset(summary.getName());
            DatasetProperties dsProperties = DatasetProperties.of(summary.getProperties());
            String dsType = summary.getType();
            Dataset dataset = null;
            try {
                try {
                    dataset = impersonator.doAs(dsInstance, new Callable<Dataset>() {

                        @Override
                        public Dataset call() throws Exception {
                            return systemDatasetInstantiator.getDataset(dsInstance);
                        }
                    });
                } catch (Exception e) {
                    LOG.warn("Exception while instantiating dataset {}", dsInstance, e);
                }
                SystemMetadataWriter writer = new DatasetSystemMetadataWriter(metadataStore, dsInstance, dsProperties, dataset, dsType, summary.getDescription());
                writer.write();
            } finally {
                if (dataset != null) {
                    dataset.close();
                }
            }
        }
    }
}

Example 7 with DatasetFramework

use of co.cask.cdap.data2.dataset2.DatasetFramework in project cdap by caskdata.

the class RemoteDatasetFrameworkTest method testSystemNamespace.

// Note: Cannot have these system namespace restrictions in system namespace since we use it internally in
// DatasetMetaTable util to add modules to system namespace. However, we should definitely impose these restrictions
// in RemoteDatasetFramework.
@Test
public void testSystemNamespace() throws DatasetManagementException {
    DatasetFramework framework = getFramework();
    // Adding module to system namespace should fail
    try {
        framework.addModule(NamespaceId.SYSTEM.datasetModule("keyValue"), new SingleTypeModule(SimpleKVTable.class));
        Assert.fail("Should not be able to add a module to system namespace");
    } catch (DatasetManagementException e) {
    // expected
    }
    try {
        framework.deleteModule(NamespaceId.SYSTEM.datasetModule("orderedTable-memory"));
        Assert.fail("Should not be able to delete a default module.");
    } catch (DatasetManagementException e) {
    // expected
    }
    try {
        framework.deleteAllModules(NamespaceId.SYSTEM);
        Assert.fail("Should not be able to delete modules from system namespace");
    } catch (DatasetManagementException e) {
    // expected
    }
}

Example 8 with DatasetFramework

use of co.cask.cdap.data2.dataset2.DatasetFramework in project cdap by caskdata.

the class NamespaceHttpHandlerTest method testDeleteDatasetsOnly.

@Test
public void testDeleteDatasetsOnly() throws Exception {
    CConfiguration cConf = getInjector().getInstance(CConfiguration.class);
    // test deleting non-existent namespace
    assertResponseCode(200, createNamespace(NAME));
    assertResponseCode(200, getNamespace(NAME));
    NamespacedLocationFactory namespacedLocationFactory = getInjector().getInstance(NamespacedLocationFactory.class);
    Location nsLocation = namespacedLocationFactory.get(new NamespaceId(NAME));
    Assert.assertTrue(nsLocation.exists());
    DatasetFramework dsFramework = getInjector().getInstance(DatasetFramework.class);
    deploy(AppWithServices.class, Constants.Gateway.API_VERSION_3_TOKEN, NAME);
    deploy(AppWithDataset.class, Constants.Gateway.API_VERSION_3_TOKEN, NAME);
    DatasetId myDataset = new DatasetId(NAME, "myds");
    Assert.assertTrue(dsFramework.hasInstance(myDataset));
    Id.Program program = Id.Program.from(NAME_ID, "AppWithServices", ProgramType.SERVICE, "NoOpService");
    startProgram(program);
    boolean resetEnabled = cConf.getBoolean(Constants.Dangerous.UNRECOVERABLE_RESET);
    cConf.setBoolean(Constants.Dangerous.UNRECOVERABLE_RESET, false);
    // because reset is not enabled
    assertResponseCode(403, deleteNamespaceData(NAME));
    Assert.assertTrue(nsLocation.exists());
    cConf.setBoolean(Constants.Dangerous.UNRECOVERABLE_RESET, resetEnabled);
    // because service is running
    assertResponseCode(409, deleteNamespaceData(NAME));
    Assert.assertTrue(nsLocation.exists());
    stopProgram(program);
    assertResponseCode(200, deleteNamespaceData(NAME));
    Assert.assertTrue(nsLocation.exists());
    Assert.assertTrue(getAppList(NAME).size() == 2);
    Assert.assertTrue(getAppDetails(NAME, "AppWithServices").get("name").getAsString().equals("AppWithServices"));
    Assert.assertTrue(getAppDetails(NAME, AppWithDataset.class.getSimpleName()).get("name").getAsString().equals(AppWithDataset.class.getSimpleName()));
    assertResponseCode(200, getNamespace(NAME));
    Assert.assertFalse(dsFramework.hasInstance(myDataset));
    assertResponseCode(200, deleteNamespace(NAME));
    assertResponseCode(404, getNamespace(NAME));
}

Example 9 with DatasetFramework

use of co.cask.cdap.data2.dataset2.DatasetFramework in project cdap by caskdata.

the class DatasetMetaTableUtil method setupDatasets.

/**
   * Adds datasets and types to the given {@link DatasetFramework} used by dataset service mds.
   *
   * @param datasetFramework framework to add types and datasets to
   */
public static void setupDatasets(DatasetFramework datasetFramework) throws IOException, DatasetManagementException {
    for (Map.Entry<String, ? extends DatasetModule> entry : getModules().entrySet()) {
        // meta tables should be in the system namespace
        DatasetModuleId moduleId = NamespaceId.SYSTEM.datasetModule(entry.getKey());
        datasetFramework.addModule(moduleId, entry.getValue());
    }
    datasetFramework.addInstance(DatasetTypeMDS.class.getName(), NamespaceId.SYSTEM.dataset(META_TABLE_NAME), DatasetProperties.EMPTY);
    datasetFramework.addInstance(DatasetInstanceMDS.class.getName(), NamespaceId.SYSTEM.dataset(INSTANCE_TABLE_NAME), DatasetProperties.EMPTY);
}

Example 10 with DatasetFramework

use of co.cask.cdap.data2.dataset2.DatasetFramework in project cdap by caskdata.

the class AbstractDatasetFrameworkTest method testCompositeDataset.

@Test
public void testCompositeDataset() throws Exception {
    // Configuring Dataset types
    DatasetFramework framework = getFramework();
    framework.addModule(IN_MEMORY, new InMemoryTableModule());
    framework.addModule(CORE, new CoreDatasetsModule());
    Assert.assertFalse(framework.hasSystemType(SimpleKVTable.class.getName()));
    Assert.assertFalse(framework.hasType(SIMPLE_KV_TYPE));
    framework.addModule(KEY_VALUE, new SingleTypeModule(SimpleKVTable.class));
    Assert.assertTrue(framework.hasType(SIMPLE_KV_TYPE));
    Assert.assertFalse(framework.hasSystemType(SimpleKVTable.class.getName()));
    // Creating instance
    Assert.assertFalse(framework.hasInstance(MY_TABLE));
    framework.addInstance(SimpleKVTable.class.getName(), MY_TABLE, DatasetProperties.EMPTY);
    Assert.assertTrue(framework.hasInstance(MY_TABLE));
    testCompositeDataset(framework);
    // cleanup
    framework.deleteInstance(MY_TABLE);
    framework.deleteModule(KEY_VALUE);
    framework.deleteModule(CORE);
    framework.deleteModule(IN_MEMORY);
}