Examples with NamespaceId io.cdap.cdap.proto.id.NamespaceId used on opensource projects

Example 51 with NamespaceId

use of io.cdap.cdap.proto.id.NamespaceId in project cdap by caskdata.

the class LineageAdminTest method testWorkflowLineage.

@Test
public void testWorkflowLineage() {
    TransactionRunner transactionRunner = getInjector().getInstance(TransactionRunner.class);
    LineageStoreReader lineageReader = new DefaultLineageStoreReader(transactionRunner);
    LineageWriter lineageWriter = new BasicLineageWriter(transactionRunner);
    ApplicationId testApp = NamespaceId.DEFAULT.app("testApp");
    ProgramId workflowId = testApp.workflow("wf1");
    // if the spark and mr job are inner jobs of workflow, they should be in the same app
    ProgramId mrId = testApp.mr("mr1");
    ProgramId sparkId = testApp.mr("spark1");
    ImmutableList<WorkflowNode> nodes = ImmutableList.of(new WorkflowActionNode("mr1", new ScheduleProgramInfo(SchedulableProgramType.MAPREDUCE, "mr1")), new WorkflowActionNode("spark1", new ScheduleProgramInfo(SchedulableProgramType.SPARK, "spark1")));
    WorkflowSpecification wfSpec = new WorkflowSpecification("test", "wf1", "", Collections.emptyMap(), nodes, Collections.emptyMap(), Collections.emptyMap());
    ApplicationSpecification appSpec = new DefaultApplicationSpecification("testApp", ProjectInfo.getVersion().toString(), "dummy app", null, NamespaceId.DEFAULT.artifact("testArtifact", "1.0").toApiArtifactId(), Collections.emptyMap(), Collections.emptyMap(), Collections.emptyMap(), Collections.emptyMap(), ImmutableMap.of(workflowId.getProgram(), wfSpec), Collections.emptyMap(), Collections.emptyMap(), Collections.emptyMap(), Collections.emptyMap());
    Store store = getInjector().getInstance(Store.class);
    store.addApplication(testApp, appSpec);
    LineageAdmin lineageAdmin = new LineageAdmin(lineageReader, store);
    // Add accesses for D3 -> P2 -> D2 -> P1 -> D1 <-> P3
    // |
    // |-> P5,
    // P1 and P2 are inner programs of the workflow
    // We need to use current time here as metadata store stores access time using current time
    ProgramRunId run1 = mrId.run(RunIds.generate(System.currentTimeMillis()).getId());
    ProgramRunId run2 = sparkId.run(RunIds.generate(System.currentTimeMillis()).getId());
    ProgramRunId run3 = program3.run(RunIds.generate(System.currentTimeMillis()).getId());
    ProgramRunId workflow = workflowId.run(RunIds.generate(System.currentTimeMillis()).getId());
    ProgramRunId run5 = program5.run(RunIds.generate(System.currentTimeMillis()).getId());
    addRuns(store, workflow);
    // only mr and spark can be inner programs
    addWorkflowRuns(store, workflow.getProgram(), workflow.getRun(), run1, run2);
    addRuns(store, run3);
    addRuns(store, run5);
    // It is okay to use current time here since access time is ignore during assertions
    lineageWriter.addAccess(run1, dataset1, AccessType.WRITE);
    lineageWriter.addAccess(run1, dataset2, AccessType.READ);
    lineageWriter.addAccess(run2, dataset2, AccessType.WRITE);
    lineageWriter.addAccess(run2, dataset3, AccessType.READ);
    lineageWriter.addAccess(run3, dataset1, AccessType.UNKNOWN, null);
    lineageWriter.addAccess(run5, dataset1, AccessType.READ, null);
    // The UNKNOWN access type will get filtered out if there is READ/WRITE. It will be preserved if it is the
    // only access type
    Lineage expectedLineage = new Lineage(ImmutableSet.of(new Relation(dataset1, workflowId, AccessType.WRITE, twillRunId(workflow)), new Relation(dataset2, workflowId, AccessType.READ, twillRunId(workflow)), new Relation(dataset2, workflowId, AccessType.WRITE, twillRunId(workflow)), new Relation(dataset3, workflowId, AccessType.READ, twillRunId(workflow)), new Relation(dataset1, program3, AccessType.UNKNOWN, twillRunId(run3)), new Relation(dataset1, program5, AccessType.READ, twillRunId(run5))));
    Lineage resultLineage = lineageAdmin.computeLineage(dataset1, 500, System.currentTimeMillis() + 10000, 100, "workflow");
    // Lineage for D1
    Assert.assertEquals(expectedLineage, resultLineage);
    resultLineage = lineageAdmin.computeLineage(dataset2, 500, System.currentTimeMillis() + 10000, 100, "workflow");
    // Lineage for D2
    Assert.assertEquals(expectedLineage, resultLineage);
    // Lineage for D1 for one level should be D2 -> P1 -> D1 <-> P3
    Lineage oneLevelLineage = lineageAdmin.computeLineage(dataset1, 500, System.currentTimeMillis() + 10000, 1, "workflow");
    Assert.assertEquals(ImmutableSet.of(new Relation(dataset1, workflowId, AccessType.WRITE, twillRunId(workflow)), new Relation(dataset2, workflowId, AccessType.READ, twillRunId(workflow)), new Relation(dataset1, program5, AccessType.READ, twillRunId(run5)), new Relation(dataset1, program3, AccessType.UNKNOWN, twillRunId(run3))), oneLevelLineage.getRelations());
    // Run tests without workflow parameter
    expectedLineage = new Lineage(ImmutableSet.of(new Relation(dataset1, mrId, AccessType.WRITE, twillRunId(run1)), new Relation(dataset2, mrId, AccessType.READ, twillRunId(run1)), new Relation(dataset2, sparkId, AccessType.WRITE, twillRunId(run2)), new Relation(dataset3, sparkId, AccessType.READ, twillRunId(run2)), new Relation(dataset1, program3, AccessType.UNKNOWN, twillRunId(run3)), new Relation(dataset1, program5, AccessType.READ, twillRunId(run5))));
    resultLineage = lineageAdmin.computeLineage(dataset1, 500, System.currentTimeMillis() + 10000, 100, null);
    // Lineage for D1
    Assert.assertEquals(expectedLineage, resultLineage);
    resultLineage = lineageAdmin.computeLineage(dataset2, 500, System.currentTimeMillis() + 10000, 100, null);
    // Lineage for D2
    Assert.assertEquals(expectedLineage, resultLineage);
    // Lineage for D1 for one level should be D2 -> P1 -> D1 <-> P3
    oneLevelLineage = lineageAdmin.computeLineage(dataset1, 500, System.currentTimeMillis() + 10000, 1, null);
    Assert.assertEquals(ImmutableSet.of(new Relation(dataset1, mrId, AccessType.WRITE, twillRunId(run1)), new Relation(dataset2, mrId, AccessType.READ, twillRunId(run1)), new Relation(dataset1, program5, AccessType.READ, twillRunId(run5)), new Relation(dataset1, program3, AccessType.UNKNOWN, twillRunId(run3))), oneLevelLineage.getRelations());
    // Assert that in a different namespace both lineage and metadata should be empty
    NamespaceId customNamespace = new NamespaceId("custom_namespace");
    DatasetId customDataset1 = customNamespace.dataset(dataset1.getEntityName());
    Assert.assertEquals(new Lineage(ImmutableSet.of()), lineageAdmin.computeLineage(customDataset1, 500, System.currentTimeMillis() + 10000, 100));
}

Example 52 with NamespaceId

use of io.cdap.cdap.proto.id.NamespaceId in project cdap by caskdata.

the class AuthorizationCLITest method testAuthorizationCLI.

@Test
public void testAuthorizationCLI() throws Exception {
    Role role = new Role("admins");
    Principal principal = new Principal("spiderman", Principal.PrincipalType.USER);
    NamespaceId namespaceId = new NamespaceId("ns1");
    CLITestBase.testCommandOutputContains(cli, String.format("create namespace %s", namespaceId.getNamespace()), String.format("Namespace '%s' created successfully", namespaceId.getNamespace()));
    // test creating role
    CLITestBase.testCommandOutputContains(cli, "create role " + role.getName(), String.format("Successfully created role '%s'", role.getName()));
    // test add role to principal
    CLITestBase.testCommandOutputContains(cli, String.format("add role %s to %s %s", role.getName(), principal.getType(), principal.getName()), String.format("Successfully added role '%s' to '%s' '%s'", role.getName(), principal.getType(), principal.getName()));
    // test listing all roles
    String output = CLITestBase.getCommandOutput(cli, "list roles");
    List<String> lines = Arrays.asList(output.split("\\r?\\n"));
    Assert.assertEquals(2, lines.size());
    // 0 is just the table headers
    Assert.assertEquals(role.getName(), lines.get(1));
    // test listing roles for a principal
    output = CLITestBase.getCommandOutput(cli, String.format("list roles for %s %s", principal.getType(), principal.getName()));
    lines = Arrays.asList(output.split("\\r?\\n"));
    Assert.assertEquals(2, lines.size());
    Assert.assertEquals(role.getName(), lines.get(1));
    // test grant permission. also tests case insensitivity of Permission and Principal.PrincipalType
    CLITestBase.testCommandOutputContains(cli, String.format("grant permissions %s on entity %s to %s %s", StandardPermission.GET.name().toLowerCase(), namespaceId.toString(), principal.getType().name().toLowerCase(), principal.getName()), String.format("Successfully granted permission(s) '%s' on entity '%s' to %s '%s'", StandardPermission.GET, namespaceId.toString(), principal.getType(), principal.getName()));
    // test grant permission for application permission (dotted syntax)
    CLITestBase.testCommandOutputContains(cli, String.format("grant permissions %s.%s on entity %s to %s %s", PermissionType.APPLICATION.name().toLowerCase(), ApplicationPermission.EXECUTE.name().toLowerCase(), namespaceId.toString(), principal.getType().name().toLowerCase(), principal.getName()), String.format("Successfully granted permission(s) '%s' on entity '%s' to %s '%s'", ApplicationPermission.EXECUTE, namespaceId.toString(), principal.getType(), principal.getName()));
    // test listing privilege
    output = CLITestBase.getCommandOutput(cli, String.format("list privileges for %s %s", principal.getType(), principal.getName()));
    lines = Stream.of(output.split("\\r?\\n")).sorted().collect(Collectors.toList());
    Assert.assertEquals(3, lines.size());
    Assert.assertArrayEquals(new String[] { namespaceId.toString(), ApplicationPermission.EXECUTE.name() }, lines.get(1).split(","));
    Assert.assertArrayEquals(new String[] { namespaceId.toString(), StandardPermission.GET.name() }, lines.get(2).split(","));
    // test revoke permissions
    CLITestBase.testCommandOutputContains(cli, String.format("revoke permissions %s on entity %s from %s %s", StandardPermission.GET, namespaceId.toString(), principal.getType(), principal.getName()), String.format("Successfully revoked permission(s) '%s' on entity '%s' for %s '%s'", StandardPermission.GET, namespaceId.toString(), principal.getType(), principal.getName()));
    // grant and perform revoke on the entity
    CLITestBase.testCommandOutputContains(cli, String.format("grant permissions %s on entity %s to %s %s", StandardPermission.GET, namespaceId.toString(), principal.getType(), principal.getName()), String.format("Successfully granted permission(s) '%s' on entity '%s' to %s '%s'", StandardPermission.GET, namespaceId.toString(), principal.getType(), principal.getName()));
    CLITestBase.testCommandOutputContains(cli, String.format("revoke all on entity %s ", namespaceId.toString()), String.format("Successfully revoked all permissions on entity '%s' for all principals", namespaceId.toString()));
    // test remove role from principal
    CLITestBase.testCommandOutputContains(cli, String.format("remove role %s from %s %s", role.getName(), principal.getType(), principal.getName()), String.format("Successfully removed role '%s' from %s '%s'", role.getName(), principal.getType(), principal.getName()));
    // test remove role (which doesn't exist) from principal
    Role nonexistentRole = new Role("nonexistent_role");
    CLITestBase.testCommandOutputContains(cli, String.format("remove role %s from %s %s", nonexistentRole.getName(), principal.getType(), principal.getName()), String.format("Error: %s not found", nonexistentRole));
}

Example 53 with NamespaceId

use of io.cdap.cdap.proto.id.NamespaceId in project cdap by caskdata.

the class DatasetTypeHandler method deleteModules.

@DELETE
@Path("/data/modules")
public void deleteModules(HttpRequest request, HttpResponder responder, @PathParam("namespace-id") String namespaceId) throws Exception {
    typeService.deleteAll(new NamespaceId(namespaceId));
    responder.sendStatus(HttpResponseStatus.OK);
}

Example 54 with NamespaceId

use of io.cdap.cdap.proto.id.NamespaceId in project cdap by caskdata.

the class DefaultDatasetTypeService method delete.

/**
 * Deletes the specified {@link DatasetModuleId}
 */
@Override
public void delete(DatasetModuleId datasetModuleId) throws Exception {
    NamespaceId namespaceId = datasetModuleId.getParent();
    if (NamespaceId.SYSTEM.equals(namespaceId)) {
        throw new UnsupportedOperationException(String.format("Cannot delete module '%s' from '%s' namespace.", datasetModuleId.getModule(), datasetModuleId.getNamespace()));
    }
    ensureNamespaceExists(namespaceId);
    DatasetModuleMeta moduleMeta = typeManager.getModule(datasetModuleId);
    if (moduleMeta == null) {
        throw new DatasetModuleNotFoundException(datasetModuleId);
    }
    try {
        typeManager.deleteModule(datasetModuleId);
    } catch (DatasetModuleConflictException e) {
        throw new DatasetModuleCannotBeDeletedException(datasetModuleId, e.getMessage());
    }
}

Example 55 with NamespaceId

use of io.cdap.cdap.proto.id.NamespaceId in project cdap by caskdata.

the class DefaultDatasetNamespace method fromNamespaced.

@Override
public DatasetId fromNamespaced(String namespaced) {
    Preconditions.checkArgument(namespaced != null, "Dataset name should not be null");
    // Dataset name is of the format <table-prefix>.<namespace>.<dataset-name>
    Preconditions.checkArgument(namespaced.startsWith(rootPrefix), "Dataset name should start with " + rootPrefix);
    // rootIndex is the index of the first character after the root prefix
    int rootIndex = rootPrefix.length();
    // namespaceIndex is the index of the first dot after the rootIndex
    int namespaceIndex = namespaced.indexOf(".", rootIndex);
    // This check implies also that namespace is non-empty
    // Also, '.' is not permitted in namespace name. So this should return the full namespace.
    Preconditions.checkArgument(namespaceIndex > rootIndex, "Dataset name is expected to be in the format %s<namespace>.<dataset-name>. Found - %s", rootPrefix, namespaced);
    NamespaceId namespace = new NamespaceId(namespaced.substring(rootIndex, namespaceIndex));
    String datasetName = namespaced.substring(namespaceIndex + 1);
    Preconditions.checkArgument(!datasetName.isEmpty(), "Dataset name is expected to be in the format %s<namespace>.<dataset-name>. Found - %s", rootPrefix, namespaced);
    return namespace.dataset(datasetName);
}