Examples with DatasetSpecificationSummary io.cdap.cdap.proto.DatasetSpecificationSummary used on opensource projects

Example 31 with DatasetSpecificationSummary

use of io.cdap.cdap.proto.DatasetSpecificationSummary in project cdap by cdapio.

the class WorkflowClientTestRun method testWorkflowClient.

@Test
public void testWorkflowClient() throws Exception {
    String keyValueTableType = "io.cdap.cdap.api.dataset.lib.KeyValueTable";
    String filesetType = "io.cdap.cdap.api.dataset.lib.FileSet";
    String outputPath = new File(TMP_FOLDER.newFolder(), "output").getAbsolutePath();
    Map<String, String> runtimeArgs = ImmutableMap.of("inputPath", createInput("input"), "outputPath", outputPath, "dataset.*.keep.local", "true");
    final WorkflowId workflowId = NamespaceId.DEFAULT.app(AppWithWorkflow.NAME).workflow(AppWithWorkflow.SampleWorkflow.NAME);
    programClient.start(workflowId, false, runtimeArgs);
    assertProgramRuns(programClient, workflowId, ProgramRunStatus.COMPLETED, 1, 120);
    List<RunRecord> workflowRuns = programClient.getProgramRuns(workflowId, ProgramRunStatus.COMPLETED.name(), 0, Long.MAX_VALUE, 10);
    Assert.assertEquals(1, workflowRuns.size());
    String runId = workflowRuns.get(0).getPid();
    ProgramRunId workflowRunId = workflowId.run(runId);
    // Invalid test scenarios
    try {
        ProgramId nonExistentWorkflowId = new ProgramId(NamespaceId.DEFAULT.getNamespace(), AppWithWorkflow.NAME, ProgramType.WORKFLOW, "NonExistentWorkflow");
        ProgramRunId nonExistentWorkflowRun = nonExistentWorkflowId.run(runId);
        workflowClient.getWorkflowToken(nonExistentWorkflowRun);
        Assert.fail("Should not find a workflow token for a non-existing workflow");
    } catch (NotFoundException expected) {
    // expected
    }
    try {
        ProgramRunId invalidRunId = workflowId.run(RunIds.generate().getId());
        workflowClient.getWorkflowToken(invalidRunId);
        Assert.fail("Should not find a workflow token for a random run id");
    } catch (NotFoundException expected) {
    // expected
    }
    // Valid test scenarios
    WorkflowTokenDetail workflowToken = workflowClient.getWorkflowToken(workflowRunId);
    Assert.assertEquals(5, workflowToken.getTokenData().size());
    workflowToken = workflowClient.getWorkflowToken(workflowRunId, WorkflowToken.Scope.SYSTEM);
    Assert.assertTrue(workflowToken.getTokenData().size() > 0);
    workflowToken = workflowClient.getWorkflowToken(workflowRunId, "start_time");
    Map<String, List<WorkflowTokenDetail.NodeValueDetail>> tokenData = workflowToken.getTokenData();
    Assert.assertEquals(AppWithWorkflow.WordCountMapReduce.NAME, tokenData.get("start_time").get(0).getNode());
    Assert.assertTrue(Long.parseLong(tokenData.get("start_time").get(0).getValue()) < System.currentTimeMillis());
    workflowToken = workflowClient.getWorkflowToken(workflowRunId, WorkflowToken.Scope.USER, "action_type");
    tokenData = workflowToken.getTokenData();
    Assert.assertEquals(AppWithWorkflow.WordCountMapReduce.NAME, tokenData.get("action_type").get(0).getNode());
    Assert.assertEquals("MapReduce", tokenData.get("action_type").get(0).getValue());
    String nodeName = AppWithWorkflow.SampleWorkflow.FIRST_ACTION;
    WorkflowTokenNodeDetail workflowTokenAtNode = workflowClient.getWorkflowTokenAtNode(workflowRunId, nodeName);
    Assert.assertEquals(AppWithWorkflow.DummyAction.TOKEN_VALUE, workflowTokenAtNode.getTokenDataAtNode().get(AppWithWorkflow.DummyAction.TOKEN_KEY));
    workflowTokenAtNode = workflowClient.getWorkflowTokenAtNode(workflowRunId, nodeName, WorkflowToken.Scope.SYSTEM);
    Assert.assertEquals(0, workflowTokenAtNode.getTokenDataAtNode().size());
    workflowTokenAtNode = workflowClient.getWorkflowTokenAtNode(workflowRunId, nodeName, AppWithWorkflow.DummyAction.TOKEN_KEY);
    Assert.assertEquals(AppWithWorkflow.DummyAction.TOKEN_VALUE, workflowTokenAtNode.getTokenDataAtNode().get(AppWithWorkflow.DummyAction.TOKEN_KEY));
    String reduceOutputRecordsCounter = "org.apache.hadoop.mapreduce.TaskCounter.REDUCE_OUTPUT_RECORDS";
    workflowTokenAtNode = workflowClient.getWorkflowTokenAtNode(workflowRunId, AppWithWorkflow.WordCountMapReduce.NAME, WorkflowToken.Scope.SYSTEM, reduceOutputRecordsCounter);
    Assert.assertEquals(6, Integer.parseInt(workflowTokenAtNode.getTokenDataAtNode().get(reduceOutputRecordsCounter)));
    Map<String, DatasetSpecificationSummary> localDatasetSummaries = workflowClient.getWorkflowLocalDatasets(workflowRunId);
    Assert.assertEquals(2, localDatasetSummaries.size());
    DatasetSpecificationSummary keyValueTableSummary = new DatasetSpecificationSummary("MyTable." + runId, keyValueTableType, ImmutableMap.of("foo", "bar"));
    Assert.assertEquals(keyValueTableSummary, localDatasetSummaries.get("MyTable"));
    DatasetSpecificationSummary filesetSummary = new DatasetSpecificationSummary("MyFile." + runId, filesetType, ImmutableMap.of("anotherFoo", "anotherBar"));
    Assert.assertEquals(filesetSummary, localDatasetSummaries.get("MyFile"));
    workflowClient.deleteWorkflowLocalDatasets(workflowRunId);
    localDatasetSummaries = workflowClient.getWorkflowLocalDatasets(workflowRunId);
    Assert.assertEquals(0, localDatasetSummaries.size());
    Map<String, WorkflowNodeStateDetail> nodeStates = workflowClient.getWorkflowNodeStates(workflowRunId);
    Assert.assertEquals(3, nodeStates.size());
    WorkflowNodeStateDetail nodeState = nodeStates.get(AppWithWorkflow.SampleWorkflow.FIRST_ACTION);
    Assert.assertTrue(AppWithWorkflow.SampleWorkflow.FIRST_ACTION.equals(nodeState.getNodeId()));
    Assert.assertTrue(NodeStatus.COMPLETED == nodeState.getNodeStatus());
    nodeState = nodeStates.get(AppWithWorkflow.SampleWorkflow.SECOND_ACTION);
    Assert.assertTrue(AppWithWorkflow.SampleWorkflow.SECOND_ACTION.equals(nodeState.getNodeId()));
    Assert.assertTrue(NodeStatus.COMPLETED == nodeState.getNodeStatus());
    nodeState = nodeStates.get(AppWithWorkflow.SampleWorkflow.WORD_COUNT_MR);
    Assert.assertTrue(AppWithWorkflow.SampleWorkflow.WORD_COUNT_MR.equals(nodeState.getNodeId()));
    Assert.assertTrue(NodeStatus.COMPLETED == nodeState.getNodeStatus());
}

Example 32 with DatasetSpecificationSummary

use of io.cdap.cdap.proto.DatasetSpecificationSummary in project cdap by cdapio.

the class GenerateClientUsageExample method datasetClient.

public void datasetClient() throws Exception {
    // Construct the client used to interact with CDAP
    DatasetClient datasetClient = new DatasetClient(clientConfig);
    // Fetch the list of datasets
    List<DatasetSpecificationSummary> datasets = datasetClient.list(NamespaceId.DEFAULT);
    // Create a dataset
    DatasetId datasetId = NamespaceId.DEFAULT.dataset("someDataset");
    datasetClient.create(datasetId, "someDatasetType");
    // Truncate a dataset
    datasetClient.truncate(datasetId);
    // Delete a dataset
    datasetClient.delete(datasetId);
}

Example 33 with DatasetSpecificationSummary

use of io.cdap.cdap.proto.DatasetSpecificationSummary in project cdap by cdapio.

the class WorkflowHttpHandler method getWorkflowLocalDatasets.

@GET
@Path("/apps/{app-id}/workflows/{workflow-id}/runs/{run-id}/localdatasets")
public void getWorkflowLocalDatasets(HttpRequest request, HttpResponder responder, @PathParam("namespace-id") String namespaceId, @PathParam("app-id") String applicationId, @PathParam("workflow-id") String workflowId, @PathParam("run-id") String runId) throws NotFoundException, DatasetManagementException, UnauthorizedException {
    WorkflowSpecification workflowSpec = getWorkflowSpecForValidRun(namespaceId, applicationId, workflowId, runId);
    Map<String, DatasetSpecificationSummary> localDatasetSummaries = new HashMap<>();
    for (Map.Entry<String, DatasetCreationSpec> localDatasetEntry : workflowSpec.getLocalDatasetSpecs().entrySet()) {
        String mappedDatasetName = localDatasetEntry.getKey() + "." + runId;
        String datasetType = localDatasetEntry.getValue().getTypeName();
        Map<String, String> datasetProperties = localDatasetEntry.getValue().getProperties().getProperties();
        if (datasetFramework.hasInstance(new DatasetId(namespaceId, mappedDatasetName))) {
            localDatasetSummaries.put(localDatasetEntry.getKey(), new DatasetSpecificationSummary(mappedDatasetName, datasetType, datasetProperties));
        }
    }
    responder.sendJson(HttpResponseStatus.OK, GSON.toJson(localDatasetSummaries));
}

Example 34 with DatasetSpecificationSummary

use of io.cdap.cdap.proto.DatasetSpecificationSummary in project cdap by cdapio.

the class HBaseDatasetMetricsReporter method report.

private void report(Map<TableId, HBaseTableUtil.TableStats> tableStats) throws IOException, UnauthorizedException {
    Map<String, String> reverseNamespaceMap = hBaseTableUtil.getHBaseToCDAPNamespaceMap();
    for (Map.Entry<TableId, HBaseTableUtil.TableStats> statEntry : tableStats.entrySet()) {
        String hbaseNamespace = statEntry.getKey().getNamespace();
        String cdapNamespace = reverseNamespaceMap.get(hbaseNamespace);
        // tableNames that doesn't start with user are ignored
        if (NamespaceId.SYSTEM.getNamespace().equals(cdapNamespace)) {
            continue;
        }
        String tableName = statEntry.getKey().getTableName();
        try {
            Collection<DatasetSpecificationSummary> instances = dsFramework.getInstances(new NamespaceId(cdapNamespace));
            for (DatasetSpecificationSummary spec : instances) {
                DatasetSpecification specification = dsFramework.getDatasetSpec(new DatasetId(cdapNamespace, spec.getName()));
                if (specification.isParent(tableName)) {
                    MetricsContext collector = metricsService.getContext(ImmutableMap.of(Constants.Metrics.Tag.NAMESPACE, cdapNamespace, Constants.Metrics.Tag.DATASET, spec.getName()));
                    collector.gauge("dataset.size.mb", statEntry.getValue().getTotalSizeMB());
                    break;
                }
            }
        } catch (DatasetManagementException | ServiceUnavailableException e) {
        // No op
        }
    }
}

Example 35 with DatasetSpecificationSummary

use of io.cdap.cdap.proto.DatasetSpecificationSummary in project cdap by cdapio.

the class WorkflowHttpHandlerTest method testLocalDatasetDeletion.

@Test
public void testLocalDatasetDeletion() throws Exception {
    String keyValueTableType = "io.cdap.cdap.api.dataset.lib.KeyValueTable";
    String filesetType = "io.cdap.cdap.api.dataset.lib.FileSet";
    Map<String, String> keyValueTableProperties = ImmutableMap.of("foo", "bar");
    Map<String, String> filesetProperties = ImmutableMap.of("anotherFoo", "anotherBar");
    deploy(WorkflowAppWithLocalDatasets.class, 200, Constants.Gateway.API_VERSION_3_TOKEN, TEST_NAMESPACE2);
    File waitFile = new File(tmpFolder.newFolder() + "/wait.file");
    File doneFile = new File(tmpFolder.newFolder() + "/done.file");
    ProgramId workflowId = new ProgramId(TEST_NAMESPACE2, WorkflowAppWithLocalDatasets.NAME, ProgramType.WORKFLOW, WorkflowAppWithLocalDatasets.WORKFLOW_NAME);
    startProgram(Id.Program.fromEntityId(workflowId), ImmutableMap.of("wait.file", waitFile.getAbsolutePath(), "done.file", doneFile.getAbsolutePath(), "dataset.*.keep.local", "true"));
    while (!waitFile.exists()) {
        TimeUnit.MILLISECONDS.sleep(50);
    }
    String runId = getRunIdOfRunningProgram(Id.Program.fromEntityId(workflowId));
    doneFile.createNewFile();
    waitState(Id.Program.fromEntityId(workflowId), ProgramStatus.STOPPED.name());
    Map<String, DatasetSpecificationSummary> localDatasetSummaries = getWorkflowLocalDatasets(workflowId, runId);
    Assert.assertEquals(2, localDatasetSummaries.size());
    DatasetSpecificationSummary keyValueTableSummary = new DatasetSpecificationSummary("MyTable." + runId, keyValueTableType, keyValueTableProperties);
    Assert.assertEquals(keyValueTableSummary, localDatasetSummaries.get("MyTable"));
    DatasetSpecificationSummary filesetSummary = new DatasetSpecificationSummary("MyFile." + runId, filesetType, filesetProperties);
    Assert.assertEquals(filesetSummary, localDatasetSummaries.get("MyFile"));
    deleteWorkflowLocalDatasets(workflowId, runId);
    localDatasetSummaries = getWorkflowLocalDatasets(workflowId, runId);
    Assert.assertEquals(0, localDatasetSummaries.size());
    waitFile = new File(tmpFolder.newFolder() + "/wait.file");
    doneFile = new File(tmpFolder.newFolder() + "/done.file");
    startProgram(Id.Program.fromEntityId(workflowId), ImmutableMap.of("wait.file", waitFile.getAbsolutePath(), "done.file", doneFile.getAbsolutePath(), "dataset.MyTable.keep.local", "true"));
    while (!waitFile.exists()) {
        TimeUnit.MILLISECONDS.sleep(50);
    }
    runId = getRunIdOfRunningProgram(Id.Program.fromEntityId(workflowId));
    doneFile.createNewFile();
    waitState(Id.Program.fromEntityId(workflowId), ProgramStatus.STOPPED.name());
    localDatasetSummaries = getWorkflowLocalDatasets(workflowId, runId);
    Assert.assertEquals(1, localDatasetSummaries.size());
    keyValueTableSummary = new DatasetSpecificationSummary("MyTable." + runId, keyValueTableType, keyValueTableProperties);
    Assert.assertEquals(keyValueTableSummary, localDatasetSummaries.get("MyTable"));
    deleteWorkflowLocalDatasets(workflowId, runId);
    localDatasetSummaries = getWorkflowLocalDatasets(workflowId, runId);
    Assert.assertEquals(0, localDatasetSummaries.size());
}