Examples with ProgramId co.cask.cdap.proto.id.ProgramId used on opensource projects

Example 36 with ProgramId

use of co.cask.cdap.proto.id.ProgramId in project cdap by caskdata.

the class LineageDataset method parseRow.

private RowKey parseRow(Row row) {
    ProgramId program;
    NamespacedEntityId data;
    RunId runId;
    MDSKey.Splitter splitter = new MDSKey(row.getRow()).split();
    char marker = (char) splitter.getInt();
    LOG.trace("Got marker {}", marker);
    switch(marker) {
        case PROGRAM_MARKER:
            program = (ProgramId) toEntityId(splitter, marker);
            // inverted start time
            splitter.skipLong();
            marker = (char) splitter.getInt();
            // data
            data = toEntityId(splitter, marker);
            runId = RunIds.fromString(splitter.getString());
            return new RowKey(program, data, runId);
        case DATASET_MARKER:
        case STREAM_MARKER:
            data = toEntityId(splitter, marker);
            // inverted start time
            splitter.skipLong();
            marker = (char) splitter.getInt();
            // program
            program = (ProgramId) toEntityId(splitter, marker);
            runId = RunIds.fromString(splitter.getString());
            return new RowKey(program, data, runId);
        default:
            throw new IllegalStateException("Invalid row with marker " + marker);
    }
}

Example 37 with ProgramId

use of co.cask.cdap.proto.id.ProgramId in project cdap by caskdata.

the class UsageDataset method unregister.

/**
 * Unregisters all usage information of an application.
 * @param applicationId application
 */
public void unregister(ApplicationId applicationId) {
    ProgramId programId = ProgramKeyMaker.getProgramId(applicationId);
    // Delete datasets associated with applicationId
    for (DatasetId datasetInstanceId : getDatasets(applicationId)) {
        deleteAll(orderedPairs.get(DATASET, PROGRAM).makeKey(datasetInstanceId, programId));
    }
    // Delete streams associated with applicationId
    for (StreamId streamId : getStreams(applicationId)) {
        deleteAll(orderedPairs.get(STREAM, PROGRAM).makeKey(streamId, programId));
    }
    // Delete all mappings for applicationId
    deleteAll(orderedPairs.get(PROGRAM, DATASET).makeScanKey(programId));
    deleteAll(orderedPairs.get(PROGRAM, STREAM).makeScanKey(programId));
}

Example 38 with ProgramId

use of co.cask.cdap.proto.id.ProgramId in project cdap by caskdata.

the class UsageDatasetTest method testAllMappings.

@Test
public void testAllMappings() throws Exception {
    final UsageDataset usageDataset = getUsageDataset("testAllMappings");
    TransactionExecutor txnl = dsFrameworkUtil.newInMemoryTransactionExecutor((TransactionAware) usageDataset);
    // Add mappings
    txnl.execute(new TransactionExecutor.Subroutine() {

        @Override
        public void apply() throws Exception {
            usageDataset.register(flow11, datasetInstance1);
            usageDataset.register(service21, datasetInstance3);
            usageDataset.register(flow12, stream1);
        }
    });
    txnl.execute(new TransactionExecutor.Subroutine() {

        @Override
        public void apply() throws Exception {
            // Verify app mappings
            Assert.assertEquals(ImmutableSet.of(datasetInstance1), usageDataset.getDatasets(flow11));
            Assert.assertEquals(ImmutableSet.<DatasetId>of(), usageDataset.getDatasets(flow12));
            Assert.assertEquals(ImmutableSet.of(datasetInstance3), usageDataset.getDatasets(service21));
            Assert.assertEquals(ImmutableSet.of(stream1), usageDataset.getStreams(flow12));
            Assert.assertEquals(ImmutableSet.<StreamId>of(), usageDataset.getStreams(flow22));
            // Verify dataset/stream mappings
            Assert.assertEquals(ImmutableSet.of(flow11), usageDataset.getPrograms(datasetInstance1));
            Assert.assertEquals(ImmutableSet.of(flow12), usageDataset.getPrograms(stream1));
            Assert.assertEquals(ImmutableSet.<ProgramId>of(), usageDataset.getPrograms(stream2));
        }
    });
    // --------- Delete app1 -----------
    txnl.execute(new TransactionExecutor.Subroutine() {

        @Override
        public void apply() throws Exception {
            usageDataset.unregister(flow11.getParent());
        }
    });
    txnl.execute(new TransactionExecutor.Subroutine() {

        @Override
        public void apply() throws Exception {
            // Verify app mappings
            Assert.assertEquals(ImmutableSet.<DatasetId>of(), usageDataset.getDatasets(flow11));
            Assert.assertEquals(ImmutableSet.<DatasetId>of(), usageDataset.getDatasets(flow12));
            Assert.assertEquals(ImmutableSet.of(datasetInstance3), usageDataset.getDatasets(service21));
            Assert.assertEquals(ImmutableSet.<StreamId>of(), usageDataset.getStreams(flow12));
            Assert.assertEquals(ImmutableSet.<StreamId>of(), usageDataset.getStreams(flow22));
            // Verify dataset/stream mappings
            Assert.assertEquals(ImmutableSet.<ProgramId>of(), usageDataset.getPrograms(datasetInstance1));
            Assert.assertEquals(ImmutableSet.<ProgramId>of(), usageDataset.getPrograms(stream1));
            Assert.assertEquals(ImmutableSet.<ProgramId>of(), usageDataset.getPrograms(stream2));
            // Verify app mappings
            Assert.assertEquals(ImmutableSet.<DatasetId>of(), usageDataset.getDatasets(flow11));
            Assert.assertEquals(ImmutableSet.<DatasetId>of(), usageDataset.getDatasets(flow12));
            Assert.assertEquals(ImmutableSet.of(datasetInstance3), usageDataset.getDatasets(service21));
            Assert.assertEquals(ImmutableSet.<StreamId>of(), usageDataset.getStreams(flow12));
            Assert.assertEquals(ImmutableSet.<StreamId>of(), usageDataset.getStreams(flow22));
            // Verify dataset/stream mappings
            Assert.assertEquals(ImmutableSet.<ProgramId>of(), usageDataset.getPrograms(datasetInstance1));
            Assert.assertEquals(ImmutableSet.<ProgramId>of(), usageDataset.getPrograms(stream1));
            Assert.assertEquals(ImmutableSet.<ProgramId>of(), usageDataset.getPrograms(stream2));
            // Verify app mappings
            Assert.assertEquals(ImmutableSet.<DatasetId>of(), usageDataset.getDatasets(flow11));
            Assert.assertEquals(ImmutableSet.<DatasetId>of(), usageDataset.getDatasets(flow12));
            Assert.assertEquals(ImmutableSet.of(datasetInstance3), usageDataset.getDatasets(service21));
            Assert.assertEquals(ImmutableSet.<StreamId>of(), usageDataset.getStreams(flow12));
            Assert.assertEquals(ImmutableSet.<StreamId>of(), usageDataset.getStreams(flow22));
            // Verify dataset/stream mappings
            Assert.assertEquals(ImmutableSet.<ProgramId>of(), usageDataset.getPrograms(datasetInstance1));
            Assert.assertEquals(ImmutableSet.<ProgramId>of(), usageDataset.getPrograms(stream1));
            Assert.assertEquals(ImmutableSet.<ProgramId>of(), usageDataset.getPrograms(stream2));
        }
    });
    // --------- Delete app2 -----------
    txnl.execute(new TransactionExecutor.Subroutine() {

        @Override
        public void apply() throws Exception {
            usageDataset.unregister(flow21.getParent());
        }
    });
    txnl.execute(new TransactionExecutor.Subroutine() {

        @Override
        public void apply() throws Exception {
            // Verify app mappings
            Assert.assertEquals(ImmutableSet.<DatasetId>of(), usageDataset.getDatasets(flow11));
            Assert.assertEquals(ImmutableSet.<DatasetId>of(), usageDataset.getDatasets(flow12));
            Assert.assertEquals(ImmutableSet.<DatasetId>of(), usageDataset.getDatasets(service21));
            Assert.assertEquals(ImmutableSet.<StreamId>of(), usageDataset.getStreams(flow12));
            Assert.assertEquals(ImmutableSet.<StreamId>of(), usageDataset.getStreams(flow22));
            // Verify dataset/stream mappings
            Assert.assertEquals(ImmutableSet.<ProgramId>of(), usageDataset.getPrograms(datasetInstance1));
            Assert.assertEquals(ImmutableSet.<ProgramId>of(), usageDataset.getPrograms(stream1));
            Assert.assertEquals(ImmutableSet.<ProgramId>of(), usageDataset.getPrograms(stream2));
        }
    });
}

Example 39 with ProgramId

use of co.cask.cdap.proto.id.ProgramId in project cdap by caskdata.

the class LineageDatasetTest method testMultipleRelations.

@Test
public void testMultipleRelations() throws Exception {
    final LineageDataset lineageDataset = getLineageDataset("testMultipleRelations");
    Assert.assertNotNull(lineageDataset);
    TransactionExecutor txnl = dsFrameworkUtil.newInMemoryTransactionExecutor((TransactionAware) lineageDataset);
    final RunId runId1 = RunIds.generate(10000);
    final RunId runId2 = RunIds.generate(20000);
    final RunId runId3 = RunIds.generate(30000);
    final RunId runId4 = RunIds.generate(40000);
    final DatasetId datasetInstance1 = NamespaceId.DEFAULT.dataset("dataset1");
    final DatasetId datasetInstance2 = NamespaceId.DEFAULT.dataset("dataset2");
    final StreamId stream1 = NamespaceId.DEFAULT.stream("stream1");
    final StreamId stream2 = NamespaceId.DEFAULT.stream("stream2");
    final ProgramId program1 = NamespaceId.DEFAULT.app("app1").flow("flow1");
    final FlowletId flowlet1 = program1.flowlet("flowlet1");
    final ProgramId program2 = NamespaceId.DEFAULT.app("app2").worker("worker2");
    final ProgramId program3 = NamespaceId.DEFAULT.app("app3").service("service3");
    final ProgramRunId run11 = program1.run(runId1.getId());
    final ProgramRunId run22 = program2.run(runId2.getId());
    final ProgramRunId run23 = program2.run(runId3.getId());
    final ProgramRunId run34 = program3.run(runId4.getId());
    final long now = System.currentTimeMillis();
    final long run11Data1AccessTime = now;
    final long run22Data2AccessTime = now + 1;
    final long run22Stream1AccessTime = now + 2;
    final long run23Stream2AccessTime = now + 1;
    final long run23Data2AccessTime = now + 3;
    // noinspection UnnecessaryLocalVariable
    txnl.execute(new TransactionExecutor.Subroutine() {

        @Override
        public void apply() throws Exception {
            lineageDataset.addAccess(run11, datasetInstance1, AccessType.READ, run11Data1AccessTime, flowlet1);
            lineageDataset.addAccess(run22, datasetInstance2, AccessType.WRITE, run22Data2AccessTime);
            lineageDataset.addAccess(run22, stream1, AccessType.READ, run22Stream1AccessTime);
            lineageDataset.addAccess(run23, stream2, AccessType.READ, run23Stream2AccessTime);
            lineageDataset.addAccess(run23, datasetInstance2, AccessType.WRITE, run23Data2AccessTime);
            lineageDataset.addAccess(run34, datasetInstance2, AccessType.READ_WRITE, System.currentTimeMillis());
            lineageDataset.addAccess(run34, stream2, AccessType.UNKNOWN, System.currentTimeMillis());
        }
    });
    txnl.execute(new TransactionExecutor.Subroutine() {

        @Override
        public void apply() throws Exception {
            Assert.assertEquals(ImmutableSet.of(new Relation(datasetInstance1, program1, AccessType.READ, runId1, ImmutableSet.of(flowlet1))), lineageDataset.getRelations(datasetInstance1, 0, 100000, Predicates.<Relation>alwaysTrue()));
            Assert.assertEquals(ImmutableSet.of(new Relation(datasetInstance2, program2, AccessType.WRITE, runId2), new Relation(datasetInstance2, program2, AccessType.WRITE, runId3), new Relation(datasetInstance2, program3, AccessType.READ_WRITE, runId4)), lineageDataset.getRelations(datasetInstance2, 0, 100000, Predicates.<Relation>alwaysTrue()));
            Assert.assertEquals(ImmutableSet.of(new Relation(stream1, program2, AccessType.READ, runId2)), lineageDataset.getRelations(stream1, 0, 100000, Predicates.<Relation>alwaysTrue()));
            Assert.assertEquals(ImmutableSet.of(new Relation(stream2, program2, AccessType.READ, runId3), new Relation(stream2, program3, AccessType.UNKNOWN, runId4)), lineageDataset.getRelations(stream2, 0, 100000, Predicates.<Relation>alwaysTrue()));
            Assert.assertEquals(ImmutableSet.of(new Relation(datasetInstance2, program2, AccessType.WRITE, runId2), new Relation(stream1, program2, AccessType.READ, runId2), new Relation(datasetInstance2, program2, AccessType.WRITE, runId3), new Relation(stream2, program2, AccessType.READ, runId3)), lineageDataset.getRelations(program2, 0, 100000, Predicates.<Relation>alwaysTrue()));
            // Reduced time range
            Assert.assertEquals(ImmutableSet.of(new Relation(datasetInstance2, program2, AccessType.WRITE, runId2), new Relation(datasetInstance2, program2, AccessType.WRITE, runId3)), lineageDataset.getRelations(datasetInstance2, 0, 35000, Predicates.<Relation>alwaysTrue()));
            Assert.assertEquals(toSet(program1, datasetInstance1), lineageDataset.getEntitiesForRun(run11));
            Assert.assertEquals(ImmutableList.of(run11Data1AccessTime), lineageDataset.getAccessTimesForRun(run11));
            Assert.assertEquals(toSet(program2, datasetInstance2, stream1), lineageDataset.getEntitiesForRun(run22));
            Assert.assertEquals(ImmutableList.of(run22Data2AccessTime, run22Stream1AccessTime), lineageDataset.getAccessTimesForRun(run22));
            Assert.assertEquals(toSet(program2, datasetInstance2, stream2), lineageDataset.getEntitiesForRun(run23));
            Assert.assertEquals(ImmutableList.of(run23Data2AccessTime, run23Stream2AccessTime), lineageDataset.getAccessTimesForRun(run23));
            Assert.assertEquals(toSet(program3, datasetInstance2, stream2), lineageDataset.getEntitiesForRun(run34));
        }
    });
}

Example 40 with ProgramId

use of co.cask.cdap.proto.id.ProgramId in project cdap by caskdata.

the class EntityIdKeyHelper method getTargetIdIdFromKey.

public static NamespacedEntityId getTargetIdIdFromKey(MDSKey.Splitter keySplitter, String type) {
    if (type.equals(TYPE_MAP.get(NamespaceId.class))) {
        String namespaceId = keySplitter.getString();
        return new NamespaceId(namespaceId);
    } else if (type.equals(TYPE_MAP.get(ProgramId.class))) {
        String namespaceId = keySplitter.getString();
        String appId = keySplitter.getString();
        String programType = keySplitter.getString();
        String programId = keySplitter.getString();
        return new ProgramId(namespaceId, appId, programType, programId);
    } else if (type.equals(TYPE_MAP.get(ApplicationId.class))) {
        String namespaceId = keySplitter.getString();
        String appId = keySplitter.getString();
        return new ApplicationId(namespaceId, appId);
    } else if (type.equals(TYPE_MAP.get(ArtifactId.class))) {
        String namespaceId = keySplitter.getString();
        String name = keySplitter.getString();
        String version = keySplitter.getString();
        return new ArtifactId(namespaceId, name, version);
    } else if (type.equals(TYPE_MAP.get(DatasetId.class))) {
        String namespaceId = keySplitter.getString();
        String instanceId = keySplitter.getString();
        return new DatasetId(namespaceId, instanceId);
    } else if (type.equals(TYPE_MAP.get(StreamId.class))) {
        String namespaceId = keySplitter.getString();
        String instanceId = keySplitter.getString();
        return new StreamId(namespaceId, instanceId);
    } else if (type.equals(TYPE_MAP.get(StreamViewId.class))) {
        String namespaceId = keySplitter.getString();
        String streamId = keySplitter.getString();
        String viewId = keySplitter.getString();
        return new StreamViewId(namespaceId, streamId, viewId);
    }
    throw new IllegalArgumentException("Illegal Type " + type + " of metadata source.");
}