Examples with DatasetSpecificationSummary io.cdap.cdap.proto.DatasetSpecificationSummary used on opensource projects

Example 26 with DatasetSpecificationSummary

use of io.cdap.cdap.proto.DatasetSpecificationSummary in project cdap by caskdata.

the class AbstractDatasetFrameworkTest method testNamespaceInstanceIsolation.

@Test
@SuppressWarnings("ConstantConditions")
public void testNamespaceInstanceIsolation() throws Exception {
    DatasetFramework framework = getFramework();
    // create 2 namespaces
    NamespaceId namespace1 = new NamespaceId("ns1");
    NamespaceId namespace2 = new NamespaceId("ns2");
    namespaceAdmin.create(new NamespaceMeta.Builder().setName(namespace1).build());
    namespaceAdmin.create(new NamespaceMeta.Builder().setName(namespace2).build());
    namespacePathLocator.get(namespace1).mkdirs();
    namespacePathLocator.get(namespace2).mkdirs();
    // create 2 tables, one in each namespace. both tables have the same name.
    DatasetId table1ID = namespace1.dataset("table");
    DatasetId table2ID = namespace2.dataset("table");
    // have slightly different properties so that we can distinguish between them
    framework.addInstance(Table.class.getName(), table1ID, DatasetProperties.builder().add("tag", "table1").build());
    framework.addInstance(Table.class.getName(), table2ID, DatasetProperties.builder().add("tag", "table2").build());
    // perform some data operations to make sure they are not the same underlying table
    final Table table1 = framework.getDataset(table1ID, Maps.<String, String>newHashMap(), null);
    final Table table2 = framework.getDataset(table2ID, Maps.<String, String>newHashMap(), null);
    TransactionExecutor txnl = new DefaultTransactionExecutor(new MinimalTxSystemClient(), (TransactionAware) table1, (TransactionAware) table2);
    txnl.execute(new TransactionExecutor.Subroutine() {

        @Override
        public void apply() throws Exception {
            table1.put(Bytes.toBytes("rowkey"), Bytes.toBytes("column"), Bytes.toBytes("val1"));
            table2.put(Bytes.toBytes("rowkey"), Bytes.toBytes("column"), Bytes.toBytes("val2"));
        }
    });
    // check data is different, which means they are different underlying tables
    txnl.execute(new TransactionExecutor.Subroutine() {

        @Override
        public void apply() throws Exception {
            Assert.assertEquals("val1", Bytes.toString(table1.get(Bytes.toBytes("rowkey"), Bytes.toBytes("column"))));
            Assert.assertEquals("val2", Bytes.toString(table2.get(Bytes.toBytes("rowkey"), Bytes.toBytes("column"))));
        }
    });
    // check get all in a namespace only includes those in that namespace
    Collection<DatasetSpecificationSummary> specs = framework.getInstances(namespace1);
    Assert.assertEquals(1, specs.size());
    Assert.assertEquals("table1", specs.iterator().next().getProperties().get("tag"));
    specs = framework.getInstances(namespace2);
    Assert.assertEquals(1, specs.size());
    Assert.assertEquals("table2", specs.iterator().next().getProperties().get("tag"));
    // delete one instance and make sure the other still exists
    framework.deleteInstance(table1ID);
    Assert.assertFalse(framework.hasInstance(table1ID));
    Assert.assertTrue(framework.hasInstance(table2ID));
    // delete all instances in one namespace and make sure the other still exists
    framework.addInstance(Table.class.getName(), table1ID, DatasetProperties.EMPTY);
    framework.deleteAllInstances(namespace1);
    Assert.assertTrue(framework.hasInstance(table2ID));
    // delete one namespace and make sure the other still exists
    namespacePathLocator.get(namespace1).delete(true);
    Assert.assertTrue(framework.hasInstance(table2ID));
}

Example 27 with DatasetSpecificationSummary

use of io.cdap.cdap.proto.DatasetSpecificationSummary in project cdap by caskdata.

the class DatasetInstanceHandlerTest method testBasics.

@Test
public void testBasics() throws Exception {
    // nothing has been created, modules and types list is empty
    List<DatasetSpecificationSummary> instances = getInstances().getResponseObject();
    // nothing in the beginning
    Assert.assertEquals(0, instances.size());
    try {
        // create dataset instance with type that is not yet known to the system should fail
        DatasetProperties props = DatasetProperties.builder().add("prop1", "val1").build();
        Assert.assertEquals(HttpStatus.SC_NOT_FOUND, createInstance("dataset1", "datasetType2", props).getResponseCode());
        // deploy modules
        deployModule("module1", TestModule1.class);
        deployModule("module2", TestModule2.class);
        // create dataset instance
        String description = "test instance description";
        HttpResponse response = createInstance("dataset1", "datasetType2", description, props);
        Assert.assertEquals(HttpStatus.SC_OK, response.getResponseCode());
        // verify module cannot be deleted which type is used for the dataset
        int modulesBeforeDelete = getModules().getResponseObject().size();
        Assert.assertEquals(HttpStatus.SC_CONFLICT, deleteModule("module2").getResponseCode());
        Assert.assertEquals(HttpStatus.SC_CONFLICT, deleteModules().getResponseCode());
        Assert.assertEquals(modulesBeforeDelete, getModules().getResponseObject().size());
        // Verify that dataset instance can be retrieved using specified properties
        Map<String, String> properties = new HashMap<>(props.getProperties());
        instances = getInstancesWithProperties(NamespaceId.DEFAULT.getNamespace(), properties).getResponseObject();
        Assert.assertEquals(1, instances.size());
        properties.put("some_prop_not_associated_with_dataset", "somevalue");
        instances = getInstancesWithProperties(NamespaceId.DEFAULT.getNamespace(), properties).getResponseObject();
        Assert.assertEquals(0, instances.size());
        // verify instance was created
        instances = getInstances().getResponseObject();
        Assert.assertEquals(1, instances.size());
        // verifying spec is same as expected
        DatasetSpecification dataset1Spec = createType2Spec("dataset1", "datasetType2", description, props);
        Assert.assertEquals(spec2Summary(dataset1Spec), instances.get(0));
        // verify created instance info can be retrieved
        DatasetMeta datasetInfo = getInstanceObject("dataset1").getResponseObject();
        Assert.assertEquals(dataset1Spec, datasetInfo.getSpec());
        Assert.assertEquals(dataset1Spec.getType(), datasetInfo.getType().getName());
        // type meta should have 2 modules that has to be loaded to create type's class
        // and in the order they must be loaded
        List<DatasetModuleMeta> modules = datasetInfo.getType().getModules();
        Assert.assertEquals(2, modules.size());
        DatasetTypeHandlerTest.verify(modules.get(0), "module1", TestModule1.class, ImmutableList.of("datasetType1"), Collections.emptyList(), ImmutableList.of("module2"));
        DatasetTypeHandlerTest.verify(modules.get(1), "module2", TestModule2.class, ImmutableList.of("datasetType2"), ImmutableList.of("module1"), Collections.emptyList());
        // try to retrieve non-existed instance
        Assert.assertEquals(HttpStatus.SC_NOT_FOUND, getInstance("non-existing-dataset").getResponseCode());
        // cannot create instance with same name again
        Assert.assertEquals(HttpStatus.SC_CONFLICT, createInstance("dataset1", "datasetType2", props).getResponseCode());
        Assert.assertEquals(1, getInstances().getResponseObject().size());
        // cannot delete non-existing dataset instance
        Assert.assertEquals(HttpStatus.SC_NOT_FOUND, deleteInstance("non-existing-dataset").getResponseCode());
        Assert.assertEquals(1, getInstances().getResponseObject().size());
        // verify creation of dataset instance with null properties
        Assert.assertEquals(HttpStatus.SC_OK, createInstance("nullPropertiesTable", "datasetType2").getResponseCode());
        // since dataset instance description is not provided, we are using the description given by the dataset type
        DatasetSpecification nullPropertiesTableSpec = createType2Spec("nullPropertiesTable", "datasetType2", TestModule2.DESCRIPTION, DatasetProperties.EMPTY);
        DatasetSpecificationSummary actualSummary = getSummaryForInstance("nullPropertiesTable", getInstances().getResponseObject());
        Assert.assertEquals(spec2Summary(nullPropertiesTableSpec), actualSummary);
        // delete dataset instance
        Assert.assertEquals(HttpStatus.SC_OK, deleteInstance("dataset1").getResponseCode());
        Assert.assertEquals(HttpStatus.SC_OK, deleteInstance("nullPropertiesTable").getResponseCode());
        Assert.assertEquals(0, getInstances().getResponseObject().size());
        // create workflow local dataset instance
        DatasetProperties localDSProperties = DatasetProperties.builder().add("prop1", "val1").add(Constants.AppFabric.WORKFLOW_LOCAL_DATASET_PROPERTY, "true").build();
        Assert.assertEquals(HttpStatus.SC_OK, createInstance("localDSInstance", "datasetType2", localDSProperties).getResponseCode());
        // getInstances call should still return 0
        Assert.assertEquals(0, getInstances().getResponseObject().size());
        Assert.assertEquals(HttpStatus.SC_OK, deleteInstance("localDSInstance").getResponseCode());
        // delete dataset modules
        Assert.assertEquals(HttpStatus.SC_OK, deleteModule("module2").getResponseCode());
        Assert.assertEquals(HttpStatus.SC_OK, deleteModule("module1").getResponseCode());
    } finally {
        deleteInstance("dataset1");
        deleteInstance("nullPropertiesTable");
        deleteInstance("localDSInstance");
        deleteModule("module2");
        deleteModule("module1");
    }
}

Example 28 with DatasetSpecificationSummary

use of io.cdap.cdap.proto.DatasetSpecificationSummary in project cdap by caskdata.

the class WorkflowHttpHandlerTest method testLocalDatasetDeletion.

@Test
public void testLocalDatasetDeletion() throws Exception {
    String keyValueTableType = "io.cdap.cdap.api.dataset.lib.KeyValueTable";
    String filesetType = "io.cdap.cdap.api.dataset.lib.FileSet";
    Map<String, String> keyValueTableProperties = ImmutableMap.of("foo", "bar");
    Map<String, String> filesetProperties = ImmutableMap.of("anotherFoo", "anotherBar");
    deploy(WorkflowAppWithLocalDatasets.class, 200, Constants.Gateway.API_VERSION_3_TOKEN, TEST_NAMESPACE2);
    File waitFile = new File(tmpFolder.newFolder() + "/wait.file");
    File doneFile = new File(tmpFolder.newFolder() + "/done.file");
    ProgramId workflowId = new ProgramId(TEST_NAMESPACE2, WorkflowAppWithLocalDatasets.NAME, ProgramType.WORKFLOW, WorkflowAppWithLocalDatasets.WORKFLOW_NAME);
    startProgram(Id.Program.fromEntityId(workflowId), ImmutableMap.of("wait.file", waitFile.getAbsolutePath(), "done.file", doneFile.getAbsolutePath(), "dataset.*.keep.local", "true"));
    while (!waitFile.exists()) {
        TimeUnit.MILLISECONDS.sleep(50);
    }
    String runId = getRunIdOfRunningProgram(Id.Program.fromEntityId(workflowId));
    doneFile.createNewFile();
    waitState(Id.Program.fromEntityId(workflowId), ProgramStatus.STOPPED.name());
    Map<String, DatasetSpecificationSummary> localDatasetSummaries = getWorkflowLocalDatasets(workflowId, runId);
    Assert.assertEquals(2, localDatasetSummaries.size());
    DatasetSpecificationSummary keyValueTableSummary = new DatasetSpecificationSummary("MyTable." + runId, keyValueTableType, keyValueTableProperties);
    Assert.assertEquals(keyValueTableSummary, localDatasetSummaries.get("MyTable"));
    DatasetSpecificationSummary filesetSummary = new DatasetSpecificationSummary("MyFile." + runId, filesetType, filesetProperties);
    Assert.assertEquals(filesetSummary, localDatasetSummaries.get("MyFile"));
    deleteWorkflowLocalDatasets(workflowId, runId);
    localDatasetSummaries = getWorkflowLocalDatasets(workflowId, runId);
    Assert.assertEquals(0, localDatasetSummaries.size());
    waitFile = new File(tmpFolder.newFolder() + "/wait.file");
    doneFile = new File(tmpFolder.newFolder() + "/done.file");
    startProgram(Id.Program.fromEntityId(workflowId), ImmutableMap.of("wait.file", waitFile.getAbsolutePath(), "done.file", doneFile.getAbsolutePath(), "dataset.MyTable.keep.local", "true"));
    while (!waitFile.exists()) {
        TimeUnit.MILLISECONDS.sleep(50);
    }
    runId = getRunIdOfRunningProgram(Id.Program.fromEntityId(workflowId));
    doneFile.createNewFile();
    waitState(Id.Program.fromEntityId(workflowId), ProgramStatus.STOPPED.name());
    localDatasetSummaries = getWorkflowLocalDatasets(workflowId, runId);
    Assert.assertEquals(1, localDatasetSummaries.size());
    keyValueTableSummary = new DatasetSpecificationSummary("MyTable." + runId, keyValueTableType, keyValueTableProperties);
    Assert.assertEquals(keyValueTableSummary, localDatasetSummaries.get("MyTable"));
    deleteWorkflowLocalDatasets(workflowId, runId);
    localDatasetSummaries = getWorkflowLocalDatasets(workflowId, runId);
    Assert.assertEquals(0, localDatasetSummaries.size());
}

Example 29 with DatasetSpecificationSummary

use of io.cdap.cdap.proto.DatasetSpecificationSummary in project cdap by caskdata.

the class RunRecordCorrectorServiceTest method testLocalDatasetDeleteion.

@Test
public void testLocalDatasetDeleteion() throws Exception {
    // Deploy an app
    deploy(WorkflowAppWithLocalDataset.class, 200, Constants.Gateway.API_VERSION_3_TOKEN, TEST_NAMESPACE1);
    final ProgramId workflow = new NamespaceId(TEST_NAMESPACE1).app(WorkflowAppWithLocalDataset.APP_NAME).workflow(WorkflowAppWithLocalDataset.WORKFLOW_NAME);
    startProgram(Id.Program.fromEntityId(workflow), ImmutableMap.of("dataset.*.keep.local", "true"));
    // Wait until we have a COMPLETED run record
    Tasks.waitFor(1, () -> getProgramRuns(Id.Program.fromEntityId(workflow), ProgramRunStatus.COMPLETED).size(), 5, TimeUnit.SECONDS);
    // Get the RunRecord
    List<RunRecord> runRecords = getProgramRuns(Id.Program.fromEntityId(workflow), ProgramRunStatus.COMPLETED);
    Assert.assertEquals(1, runRecords.size());
    String pid = runRecords.get(0).getPid();
    // Get the local dataset specifications
    final Map<String, String> properties = ImmutableMap.of(Constants.AppFabric.WORKFLOW_LOCAL_DATASET_PROPERTY, "true");
    Collection<DatasetSpecificationSummary> instances = datasetFramework.getInstances(new NamespaceId(TEST_NAMESPACE1), properties);
    Assert.assertEquals(1, instances.size());
    DatasetSpecificationSummary summary = instances.iterator().next();
    Assert.assertTrue(summary.getName().endsWith(pid));
    // Update the dataset properties to remove keep.local so that local dataset deleter can delete it
    Map<String, String> updatedProperties = new HashMap<>(summary.getProperties());
    updatedProperties.remove(Constants.AppFabric.WORKFLOW_KEEP_LOCAL);
    datasetFramework.updateInstance(new DatasetId(TEST_NAMESPACE1, summary.getName()), DatasetProperties.of(updatedProperties));
    // Start the local dataset deletion service now
    CConfiguration testConf = CConfiguration.create();
    // set threshold to 0 so that it will actually correct the record
    testConf.set(Constants.AppFabric.LOCAL_DATASET_DELETER_INTERVAL_SECONDS, "1");
    testConf.set(Constants.AppFabric.LOCAL_DATASET_DELETER_INITIAL_DELAY_SECONDS, "1");
    new LocalRunRecordCorrectorService(testConf, store, programStateWriter, runtimeService, namespaceAdmin, datasetFramework).startUp();
    // 
    // Wait for the deletion of the local dataset
    Tasks.waitFor(0, () -> datasetFramework.getInstances(new NamespaceId(TEST_NAMESPACE1), properties).size(), 30, TimeUnit.SECONDS, 1, TimeUnit.SECONDS);
}

Example 30 with DatasetSpecificationSummary

use of io.cdap.cdap.proto.DatasetSpecificationSummary in project cdap by caskdata.

the class DatasetUpgrader method upgradeSystemDatasets.

private void upgradeSystemDatasets(ExecutorService executor) throws Exception {
    Map<String, Future<?>> futures = new HashMap<>();
    for (final DatasetSpecificationSummary spec : dsFramework.getInstances(NamespaceId.SYSTEM)) {
        final DatasetId datasetId = NamespaceId.SYSTEM.dataset(spec.getName());
        Runnable runnable = new Runnable() {

            @Override
            public void run() {
                try {
                    LOG.info("Upgrading dataset in system namespace: {}, spec: {}", spec.getName(), spec.toString());
                    DatasetAdmin admin = dsFramework.getAdmin(datasetId, null);
                    // we know admin is not null, since we are looping over existing datasets
                    // noinspection ConstantConditions
                    admin.upgrade();
                    LOG.info("Upgraded dataset: {}", spec.getName());
                } catch (Exception e) {
                    throw new RuntimeException(e);
                }
            }
        };
        Future<?> future = executor.submit(runnable);
        futures.put(datasetId.toString(), future);
    }
    // Wait for the system dataset upgrades to complete
    Map<String, Throwable> failed = waitForUpgrade(futures);
    if (!failed.isEmpty()) {
        for (Map.Entry<String, Throwable> entry : failed.entrySet()) {
            LOG.error("Failed to upgrade system dataset {}", entry.getKey(), entry.getValue());
        }
        throw new Exception(String.format("Error upgrading system datasets. %s of %s failed", failed.size(), futures.size()));
    }
}