Examples with DatasetSpecificationSummary co.cask.cdap.proto.DatasetSpecificationSummary used on opensource projects

Example 11 with DatasetSpecificationSummary

use of co.cask.cdap.proto.DatasetSpecificationSummary in project cdap by caskdata.

the class GenerateClientUsageExample method datasetClient.

public void datasetClient() throws Exception {
    // Construct the client used to interact with CDAP
    DatasetClient datasetClient = new DatasetClient(clientConfig);
    // Fetch the list of datasets
    List<DatasetSpecificationSummary> datasets = datasetClient.list(NamespaceId.DEFAULT);
    // Create a dataset
    DatasetId datasetId = NamespaceId.DEFAULT.dataset("someDataset");
    datasetClient.create(datasetId, "someDatasetType");
    // Truncate a dataset
    datasetClient.truncate(datasetId);
    // Delete a dataset
    datasetClient.delete(datasetId);
}

Example 12 with DatasetSpecificationSummary

use of co.cask.cdap.proto.DatasetSpecificationSummary in project cdap by caskdata.

the class DatasetUpgrader method upgradeSystemDatasets.

private void upgradeSystemDatasets(ExecutorService executor) throws Exception {
    Map<String, Future<?>> futures = new HashMap<>();
    for (final DatasetSpecificationSummary spec : dsFramework.getInstances(NamespaceId.SYSTEM)) {
        final DatasetId datasetId = NamespaceId.SYSTEM.dataset(spec.getName());
        Runnable runnable = new Runnable() {

            public void run() {
                try {
                    LOG.info("Upgrading dataset in system namespace: {}, spec: {}", spec.getName(), spec.toString());
                    DatasetAdmin admin = dsFramework.getAdmin(datasetId, null);
                    // we know admin is not null, since we are looping over existing datasets
                    //noinspection ConstantConditions
                    admin.upgrade();
                    LOG.info("Upgraded dataset: {}", spec.getName());
                } catch (Exception e) {
                    throw new RuntimeException(e);
                }
            }
        };
        Future<?> future = executor.submit(runnable);
        futures.put(datasetId.toString(), future);
    }
    // Wait for the system dataset upgrades to complete
    Map<String, Throwable> failed = waitForUpgrade(futures);
    if (!failed.isEmpty()) {
        for (Map.Entry<String, Throwable> entry : failed.entrySet()) {
            LOG.error("Failed to upgrade system dataset {}", entry.getKey(), entry.getValue());
        }
        throw new Exception(String.format("Error upgrading system datasets. %s of %s failed", failed.size(), futures.size()));
    }
}

Example 13 with DatasetSpecificationSummary

use of co.cask.cdap.proto.DatasetSpecificationSummary in project cdap by caskdata.

the class WorkflowHttpHandlerTest method testLocalDatasetDeletion.

@Test
public void testLocalDatasetDeletion() throws Exception {
    String keyValueTableType = "co.cask.cdap.api.dataset.lib.KeyValueTable";
    String filesetType = "co.cask.cdap.api.dataset.lib.FileSet";
    Map<String, String> keyValueTableProperties = ImmutableMap.of("foo", "bar");
    Map<String, String> filesetProperties = ImmutableMap.of("anotherFoo", "anotherBar");
    HttpResponse response = deploy(WorkflowAppWithLocalDatasets.class, Constants.Gateway.API_VERSION_3_TOKEN, TEST_NAMESPACE2);
    Assert.assertEquals(200, response.getStatusLine().getStatusCode());
    File waitFile = new File(tmpFolder.newFolder() + "/wait.file");
    File doneFile = new File(tmpFolder.newFolder() + "/done.file");
    ProgramId workflowId = new ProgramId(TEST_NAMESPACE2, WorkflowAppWithLocalDatasets.NAME, ProgramType.WORKFLOW, WorkflowAppWithLocalDatasets.WORKFLOW_NAME);
    startProgram(workflowId.toId(), ImmutableMap.of("wait.file", waitFile.getAbsolutePath(), "done.file", doneFile.getAbsolutePath(), "dataset.*.keep.local", "true"));
    while (!waitFile.exists()) {
        TimeUnit.MILLISECONDS.sleep(50);
    }
    String runId = getRunIdOfRunningProgram(workflowId.toId());
    doneFile.createNewFile();
    waitState(workflowId.toId(), ProgramStatus.STOPPED.name());
    Map<String, DatasetSpecificationSummary> localDatasetSummaries = getWorkflowLocalDatasets(workflowId, runId);
    Assert.assertEquals(2, localDatasetSummaries.size());
    DatasetSpecificationSummary keyValueTableSummary = new DatasetSpecificationSummary("MyTable." + runId, keyValueTableType, keyValueTableProperties);
    Assert.assertEquals(keyValueTableSummary, localDatasetSummaries.get("MyTable"));
    DatasetSpecificationSummary filesetSummary = new DatasetSpecificationSummary("MyFile." + runId, filesetType, filesetProperties);
    Assert.assertEquals(filesetSummary, localDatasetSummaries.get("MyFile"));
    deleteWorkflowLocalDatasets(workflowId, runId);
    localDatasetSummaries = getWorkflowLocalDatasets(workflowId, runId);
    Assert.assertEquals(0, localDatasetSummaries.size());
    waitFile = new File(tmpFolder.newFolder() + "/wait.file");
    doneFile = new File(tmpFolder.newFolder() + "/done.file");
    startProgram(workflowId.toId(), ImmutableMap.of("wait.file", waitFile.getAbsolutePath(), "done.file", doneFile.getAbsolutePath(), "dataset.MyTable.keep.local", "true"));
    while (!waitFile.exists()) {
        TimeUnit.MILLISECONDS.sleep(50);
    }
    runId = getRunIdOfRunningProgram(workflowId.toId());
    doneFile.createNewFile();
    waitState(workflowId.toId(), ProgramStatus.STOPPED.name());
    localDatasetSummaries = getWorkflowLocalDatasets(workflowId, runId);
    Assert.assertEquals(1, localDatasetSummaries.size());
    keyValueTableSummary = new DatasetSpecificationSummary("MyTable." + runId, keyValueTableType, keyValueTableProperties);
    Assert.assertEquals(keyValueTableSummary, localDatasetSummaries.get("MyTable"));
    deleteWorkflowLocalDatasets(workflowId, runId);
    localDatasetSummaries = getWorkflowLocalDatasets(workflowId, runId);
    Assert.assertEquals(0, localDatasetSummaries.size());
}

Example 14 with DatasetSpecificationSummary

use of co.cask.cdap.proto.DatasetSpecificationSummary in project cdap by caskdata.

the class WorkflowClientTestRun method testWorkflowClient.

@Test
public void testWorkflowClient() throws Exception {
    String keyValueTableType = "co.cask.cdap.api.dataset.lib.KeyValueTable";
    String filesetType = "co.cask.cdap.api.dataset.lib.FileSet";
    String outputPath = new File(TMP_FOLDER.newFolder(), "output").getAbsolutePath();
    Map<String, String> runtimeArgs = ImmutableMap.of("inputPath", createInput("input"), "outputPath", outputPath, "dataset.*.keep.local", "true");
    final WorkflowId workflowId = NamespaceId.DEFAULT.app(AppWithWorkflow.NAME).workflow(AppWithWorkflow.SampleWorkflow.NAME);
    programClient.start(workflowId, false, runtimeArgs);
    programClient.waitForStatus(workflowId, ProgramStatus.STOPPED, 60, TimeUnit.SECONDS);
    Tasks.waitFor(1, new Callable<Integer>() {

        @Override
        public Integer call() throws Exception {
            return programClient.getProgramRuns(workflowId, ProgramRunStatus.COMPLETED.name(), 0, Long.MAX_VALUE, 10).size();
        }
    }, 10, TimeUnit.SECONDS, 100, TimeUnit.MILLISECONDS);
    List<RunRecord> workflowRuns = programClient.getProgramRuns(workflowId, ProgramRunStatus.COMPLETED.name(), 0, Long.MAX_VALUE, 10);
    Assert.assertEquals(1, workflowRuns.size());
    String runId = workflowRuns.get(0).getPid();
    ProgramRunId workflowRunId = workflowId.run(runId);
    // Invalid test scenarios
    try {
        ProgramId nonExistentWorkflowId = new ProgramId(NamespaceId.DEFAULT.getNamespace(), AppWithWorkflow.NAME, ProgramType.WORKFLOW, "NonExistentWorkflow");
        ProgramRunId nonExistentWorkflowRun = nonExistentWorkflowId.run(runId);
        workflowClient.getWorkflowToken(nonExistentWorkflowRun);
        Assert.fail("Should not find a workflow token for a non-existing workflow");
    } catch (NotFoundException expected) {
    // expected
    }
    try {
        ProgramRunId invalidRunId = workflowId.run(RunIds.generate().getId());
        workflowClient.getWorkflowToken(invalidRunId);
        Assert.fail("Should not find a workflow token for a random run id");
    } catch (NotFoundException expected) {
    // expected
    }
    // Valid test scenarios
    WorkflowTokenDetail workflowToken = workflowClient.getWorkflowToken(workflowRunId);
    Assert.assertEquals(5, workflowToken.getTokenData().size());
    workflowToken = workflowClient.getWorkflowToken(workflowRunId, WorkflowToken.Scope.SYSTEM);
    Assert.assertTrue(workflowToken.getTokenData().size() > 0);
    workflowToken = workflowClient.getWorkflowToken(workflowRunId, "start_time");
    Map<String, List<WorkflowTokenDetail.NodeValueDetail>> tokenData = workflowToken.getTokenData();
    Assert.assertEquals(AppWithWorkflow.WordCountMapReduce.NAME, tokenData.get("start_time").get(0).getNode());
    Assert.assertTrue(Long.parseLong(tokenData.get("start_time").get(0).getValue()) < System.currentTimeMillis());
    workflowToken = workflowClient.getWorkflowToken(workflowRunId, WorkflowToken.Scope.USER, "action_type");
    tokenData = workflowToken.getTokenData();
    Assert.assertEquals(AppWithWorkflow.WordCountMapReduce.NAME, tokenData.get("action_type").get(0).getNode());
    Assert.assertEquals("MapReduce", tokenData.get("action_type").get(0).getValue());
    String nodeName = AppWithWorkflow.SampleWorkflow.FIRST_ACTION;
    WorkflowTokenNodeDetail workflowTokenAtNode = workflowClient.getWorkflowTokenAtNode(workflowRunId, nodeName);
    Assert.assertEquals(AppWithWorkflow.DummyAction.TOKEN_VALUE, workflowTokenAtNode.getTokenDataAtNode().get(AppWithWorkflow.DummyAction.TOKEN_KEY));
    workflowTokenAtNode = workflowClient.getWorkflowTokenAtNode(workflowRunId, nodeName, WorkflowToken.Scope.SYSTEM);
    Assert.assertEquals(0, workflowTokenAtNode.getTokenDataAtNode().size());
    workflowTokenAtNode = workflowClient.getWorkflowTokenAtNode(workflowRunId, nodeName, AppWithWorkflow.DummyAction.TOKEN_KEY);
    Assert.assertEquals(AppWithWorkflow.DummyAction.TOKEN_VALUE, workflowTokenAtNode.getTokenDataAtNode().get(AppWithWorkflow.DummyAction.TOKEN_KEY));
    String reduceOutputRecordsCounter = "org.apache.hadoop.mapreduce.TaskCounter.REDUCE_OUTPUT_RECORDS";
    workflowTokenAtNode = workflowClient.getWorkflowTokenAtNode(workflowRunId, AppWithWorkflow.WordCountMapReduce.NAME, WorkflowToken.Scope.SYSTEM, reduceOutputRecordsCounter);
    Assert.assertEquals(6, Integer.parseInt(workflowTokenAtNode.getTokenDataAtNode().get(reduceOutputRecordsCounter)));
    Map<String, DatasetSpecificationSummary> localDatasetSummaries = workflowClient.getWorkflowLocalDatasets(workflowRunId);
    Assert.assertEquals(2, localDatasetSummaries.size());
    DatasetSpecificationSummary keyValueTableSummary = new DatasetSpecificationSummary("MyTable." + runId, keyValueTableType, ImmutableMap.of("foo", "bar"));
    Assert.assertEquals(keyValueTableSummary, localDatasetSummaries.get("MyTable"));
    DatasetSpecificationSummary filesetSummary = new DatasetSpecificationSummary("MyFile." + runId, filesetType, ImmutableMap.of("anotherFoo", "anotherBar"));
    Assert.assertEquals(filesetSummary, localDatasetSummaries.get("MyFile"));
    workflowClient.deleteWorkflowLocalDatasets(workflowRunId);
    localDatasetSummaries = workflowClient.getWorkflowLocalDatasets(workflowRunId);
    Assert.assertEquals(0, localDatasetSummaries.size());
    Map<String, WorkflowNodeStateDetail> nodeStates = workflowClient.getWorkflowNodeStates(workflowRunId);
    Assert.assertEquals(3, nodeStates.size());
    WorkflowNodeStateDetail nodeState = nodeStates.get(AppWithWorkflow.SampleWorkflow.FIRST_ACTION);
    Assert.assertTrue(AppWithWorkflow.SampleWorkflow.FIRST_ACTION.equals(nodeState.getNodeId()));
    Assert.assertTrue(NodeStatus.COMPLETED == nodeState.getNodeStatus());
    nodeState = nodeStates.get(AppWithWorkflow.SampleWorkflow.SECOND_ACTION);
    Assert.assertTrue(AppWithWorkflow.SampleWorkflow.SECOND_ACTION.equals(nodeState.getNodeId()));
    Assert.assertTrue(NodeStatus.COMPLETED == nodeState.getNodeStatus());
    nodeState = nodeStates.get(AppWithWorkflow.SampleWorkflow.WORD_COUNT_MR);
    Assert.assertTrue(AppWithWorkflow.SampleWorkflow.WORD_COUNT_MR.equals(nodeState.getNodeId()));
    Assert.assertTrue(NodeStatus.COMPLETED == nodeState.getNodeStatus());
}

Example 15 with DatasetSpecificationSummary

use of co.cask.cdap.proto.DatasetSpecificationSummary in project cdap by caskdata.

the class DatasetInstanceHandlerTest method testUpdateInstance.

@Test
public void testUpdateInstance() throws Exception {
    // nothing has been created, modules and types list is empty
    List<DatasetSpecificationSummary> instances = getInstances().getResponseObject();
    // nothing in the beginning
    Assert.assertEquals(0, instances.size());
    try {
        DatasetProperties props = DatasetProperties.builder().add("prop1", "val1").add(TestModule2.NOT_RECONFIGURABLE, "this").build();
        // deploy modules
        deployModule("module1", TestModule1.class);
        deployModule("module2", TestModule2.class);
        // create dataset instance
        Assert.assertEquals(HttpStatus.SC_OK, createInstance("dataset1", "datasetType2", props).getResponseCode());
        // verify instance was created
        instances = getInstances().getResponseObject();
        Assert.assertEquals(1, instances.size());
        DatasetMeta meta = getInstanceObject("dataset1").getResponseObject();
        Map<String, String> storedOriginalProps = meta.getSpec().getOriginalProperties();
        Assert.assertEquals(props.getProperties(), storedOriginalProps);
        Map<String, String> retrievedProps = getInstanceProperties("dataset1").getResponseObject();
        Assert.assertEquals(props.getProperties(), retrievedProps);
        // these properties are incompatible because TestModule1.NOT_RECONFIGURABLE may not change
        DatasetProperties newProps = DatasetProperties.builder().add("prop2", "val2").add(TestModule2.NOT_RECONFIGURABLE, "that").build();
        Assert.assertEquals(HttpStatus.SC_CONFLICT, updateInstance("dataset1", newProps).getResponseCode());
        // update dataset instance with valid properties
        newProps = DatasetProperties.builder().add("prop2", "val2").add(TestModule2.NOT_RECONFIGURABLE, "this").build();
        Assert.assertEquals(HttpStatus.SC_OK, updateInstance("dataset1", newProps).getResponseCode());
        meta = getInstanceObject("dataset1").getResponseObject();
        Assert.assertEquals(newProps.getProperties(), meta.getSpec().getOriginalProperties());
        Assert.assertEquals("val2", meta.getSpec().getProperty("prop2"));
        Assert.assertNull(meta.getSpec().getProperty("prop1"));
        retrievedProps = getInstanceProperties("dataset1").getResponseObject();
        Assert.assertEquals(newProps.getProperties(), retrievedProps);
    } finally {
        // delete dataset instance
        Assert.assertEquals(HttpStatus.SC_OK, deleteInstance("dataset1").getResponseCode());
        Assert.assertEquals(0, getInstances().getResponseObject().size());
        // delete dataset modules
        Assert.assertEquals(HttpStatus.SC_OK, deleteModule("module2").getResponseCode());
        Assert.assertEquals(HttpStatus.SC_OK, deleteModule("module1").getResponseCode());
    }
}