Examples with DatasetId co.cask.cdap.proto.id.DatasetId used on opensource projects

Example 81 with DatasetId

use of co.cask.cdap.proto.id.DatasetId in project cdap by caskdata.

the class AuditPublishersTest method testPublishingAccessLogs.

@Test
public void testPublishingAccessLogs() {
    String datasetName = "dummyDataset";
    String datasetName2 = "dummyDataset2";
    String appName = "dummyApp";
    String workerName = "dummyWorker";
    String workerName2 = "dummyWorker2";
    InMemoryAuditPublisher auditPublisher = new InMemoryAuditPublisher();
    ProgramId workerId = new ProgramId(NamespaceId.DEFAULT.getNamespace(), appName, ProgramType.WORKER, workerName);
    DatasetId datasetId = NamespaceId.DEFAULT.dataset(datasetName);
    AuditPublishers.publishAccess(auditPublisher, datasetId, AccessType.READ_WRITE, workerId);
    List<AuditMessage> messages = auditPublisher.popMessages();
    // Since it is a READ_WRITE access, two messages are expected
    Assert.assertEquals(2, messages.size());
    // Same access so no message should be published
    AuditPublishers.publishAccess(auditPublisher, datasetId, AccessType.READ_WRITE, workerId);
    messages = auditPublisher.popMessages();
    Assert.assertEquals(0, messages.size());
    // Different accesstype, hence a message should be published
    AuditPublishers.publishAccess(auditPublisher, datasetId, AccessType.READ, workerId);
    messages = auditPublisher.popMessages();
    Assert.assertEquals(1, messages.size());
    // Different dataset name, hence a message should be published
    datasetId = NamespaceId.DEFAULT.dataset(datasetName2);
    AuditPublishers.publishAccess(auditPublisher, datasetId, AccessType.READ_WRITE, workerId);
    messages = auditPublisher.popMessages();
    Assert.assertEquals(2, messages.size());
    // Different worker name, hence a message should be published
    workerId = new ProgramId(NamespaceId.DEFAULT.getNamespace(), appName, ProgramType.WORKER, workerName2);
    AuditPublishers.publishAccess(auditPublisher, datasetId, AccessType.READ_WRITE, workerId);
    messages = auditPublisher.popMessages();
    Assert.assertEquals(2, messages.size());
}

Example 82 with DatasetId

use of co.cask.cdap.proto.id.DatasetId in project cdap by caskdata.

the class EntityIdKeyHelper method getTargetIdIdFromKey.

public static NamespacedEntityId getTargetIdIdFromKey(MDSKey.Splitter keySplitter, String type) {
    if (type.equals(TYPE_MAP.get(NamespaceId.class))) {
        String namespaceId = keySplitter.getString();
        return new NamespaceId(namespaceId);
    } else if (type.equals(TYPE_MAP.get(ProgramId.class))) {
        String namespaceId = keySplitter.getString();
        String appId = keySplitter.getString();
        String programType = keySplitter.getString();
        String programId = keySplitter.getString();
        return new ProgramId(namespaceId, appId, programType, programId);
    } else if (type.equals(TYPE_MAP.get(ApplicationId.class))) {
        String namespaceId = keySplitter.getString();
        String appId = keySplitter.getString();
        return new ApplicationId(namespaceId, appId);
    } else if (type.equals(TYPE_MAP.get(ArtifactId.class))) {
        String namespaceId = keySplitter.getString();
        String name = keySplitter.getString();
        String version = keySplitter.getString();
        return new ArtifactId(namespaceId, name, version);
    } else if (type.equals(TYPE_MAP.get(DatasetId.class))) {
        String namespaceId = keySplitter.getString();
        String instanceId = keySplitter.getString();
        return new DatasetId(namespaceId, instanceId);
    } else if (type.equals(TYPE_MAP.get(StreamId.class))) {
        String namespaceId = keySplitter.getString();
        String instanceId = keySplitter.getString();
        return new StreamId(namespaceId, instanceId);
    } else if (type.equals(TYPE_MAP.get(StreamViewId.class))) {
        String namespaceId = keySplitter.getString();
        String streamId = keySplitter.getString();
        String viewId = keySplitter.getString();
        return new StreamViewId(namespaceId, streamId, viewId);
    }
    throw new IllegalArgumentException("Illegal Type " + type + " of metadata source.");
}

Example 83 with DatasetId

use of co.cask.cdap.proto.id.DatasetId in project cdap by caskdata.

the class UsageDatasetTest method testAllMappings.

@Test
public void testAllMappings() throws Exception {
    final UsageDataset usageDataset = getUsageDataset("testAllMappings");
    TransactionExecutor txnl = dsFrameworkUtil.newInMemoryTransactionExecutor((TransactionAware) usageDataset);
    // Add mappings
    txnl.execute(new TransactionExecutor.Subroutine() {

        @Override
        public void apply() throws Exception {
            usageDataset.register(flow11, datasetInstance1);
            usageDataset.register(service21, datasetInstance3);
            usageDataset.register(flow12, stream1);
        }
    });
    txnl.execute(new TransactionExecutor.Subroutine() {

        @Override
        public void apply() throws Exception {
            // Verify app mappings
            Assert.assertEquals(ImmutableSet.of(datasetInstance1), usageDataset.getDatasets(flow11));
            Assert.assertEquals(ImmutableSet.<DatasetId>of(), usageDataset.getDatasets(flow12));
            Assert.assertEquals(ImmutableSet.of(datasetInstance3), usageDataset.getDatasets(service21));
            Assert.assertEquals(ImmutableSet.of(stream1), usageDataset.getStreams(flow12));
            Assert.assertEquals(ImmutableSet.<StreamId>of(), usageDataset.getStreams(flow22));
            // Verify dataset/stream mappings
            Assert.assertEquals(ImmutableSet.of(flow11), usageDataset.getPrograms(datasetInstance1));
            Assert.assertEquals(ImmutableSet.of(flow12), usageDataset.getPrograms(stream1));
            Assert.assertEquals(ImmutableSet.<ProgramId>of(), usageDataset.getPrograms(stream2));
        }
    });
    // --------- Delete app1 -----------
    txnl.execute(new TransactionExecutor.Subroutine() {

        @Override
        public void apply() throws Exception {
            usageDataset.unregister(flow11.getParent());
        }
    });
    txnl.execute(new TransactionExecutor.Subroutine() {

        @Override
        public void apply() throws Exception {
            // Verify app mappings
            Assert.assertEquals(ImmutableSet.<DatasetId>of(), usageDataset.getDatasets(flow11));
            Assert.assertEquals(ImmutableSet.<DatasetId>of(), usageDataset.getDatasets(flow12));
            Assert.assertEquals(ImmutableSet.of(datasetInstance3), usageDataset.getDatasets(service21));
            Assert.assertEquals(ImmutableSet.<StreamId>of(), usageDataset.getStreams(flow12));
            Assert.assertEquals(ImmutableSet.<StreamId>of(), usageDataset.getStreams(flow22));
            // Verify dataset/stream mappings
            Assert.assertEquals(ImmutableSet.<ProgramId>of(), usageDataset.getPrograms(datasetInstance1));
            Assert.assertEquals(ImmutableSet.<ProgramId>of(), usageDataset.getPrograms(stream1));
            Assert.assertEquals(ImmutableSet.<ProgramId>of(), usageDataset.getPrograms(stream2));
            // Verify app mappings
            Assert.assertEquals(ImmutableSet.<DatasetId>of(), usageDataset.getDatasets(flow11));
            Assert.assertEquals(ImmutableSet.<DatasetId>of(), usageDataset.getDatasets(flow12));
            Assert.assertEquals(ImmutableSet.of(datasetInstance3), usageDataset.getDatasets(service21));
            Assert.assertEquals(ImmutableSet.<StreamId>of(), usageDataset.getStreams(flow12));
            Assert.assertEquals(ImmutableSet.<StreamId>of(), usageDataset.getStreams(flow22));
            // Verify dataset/stream mappings
            Assert.assertEquals(ImmutableSet.<ProgramId>of(), usageDataset.getPrograms(datasetInstance1));
            Assert.assertEquals(ImmutableSet.<ProgramId>of(), usageDataset.getPrograms(stream1));
            Assert.assertEquals(ImmutableSet.<ProgramId>of(), usageDataset.getPrograms(stream2));
            // Verify app mappings
            Assert.assertEquals(ImmutableSet.<DatasetId>of(), usageDataset.getDatasets(flow11));
            Assert.assertEquals(ImmutableSet.<DatasetId>of(), usageDataset.getDatasets(flow12));
            Assert.assertEquals(ImmutableSet.of(datasetInstance3), usageDataset.getDatasets(service21));
            Assert.assertEquals(ImmutableSet.<StreamId>of(), usageDataset.getStreams(flow12));
            Assert.assertEquals(ImmutableSet.<StreamId>of(), usageDataset.getStreams(flow22));
            // Verify dataset/stream mappings
            Assert.assertEquals(ImmutableSet.<ProgramId>of(), usageDataset.getPrograms(datasetInstance1));
            Assert.assertEquals(ImmutableSet.<ProgramId>of(), usageDataset.getPrograms(stream1));
            Assert.assertEquals(ImmutableSet.<ProgramId>of(), usageDataset.getPrograms(stream2));
        }
    });
    // --------- Delete app2 -----------
    txnl.execute(new TransactionExecutor.Subroutine() {

        @Override
        public void apply() throws Exception {
            usageDataset.unregister(flow21.getParent());
        }
    });
    txnl.execute(new TransactionExecutor.Subroutine() {

        @Override
        public void apply() throws Exception {
            // Verify app mappings
            Assert.assertEquals(ImmutableSet.<DatasetId>of(), usageDataset.getDatasets(flow11));
            Assert.assertEquals(ImmutableSet.<DatasetId>of(), usageDataset.getDatasets(flow12));
            Assert.assertEquals(ImmutableSet.<DatasetId>of(), usageDataset.getDatasets(service21));
            Assert.assertEquals(ImmutableSet.<StreamId>of(), usageDataset.getStreams(flow12));
            Assert.assertEquals(ImmutableSet.<StreamId>of(), usageDataset.getStreams(flow22));
            // Verify dataset/stream mappings
            Assert.assertEquals(ImmutableSet.<ProgramId>of(), usageDataset.getPrograms(datasetInstance1));
            Assert.assertEquals(ImmutableSet.<ProgramId>of(), usageDataset.getPrograms(stream1));
            Assert.assertEquals(ImmutableSet.<ProgramId>of(), usageDataset.getPrograms(stream2));
        }
    });
}

Example 84 with DatasetId

use of co.cask.cdap.proto.id.DatasetId in project cdap by caskdata.

the class UsageRegistryTest method testUsageRegistry.

@Test
public void testUsageRegistry() {
    // instantiate a usage registry
    UsageRegistry registry = new DefaultUsageRegistry(new TransactionExecutorFactory() {

        @Override
        public TransactionExecutor createExecutor(Iterable<TransactionAware> iterable) {
            return dsFrameworkUtil.newInMemoryTransactionExecutor(iterable);
        }
    }, new ForwardingDatasetFramework(dsFrameworkUtil.getFramework()) {

        @Nullable
        @Override
        public <T extends Dataset> T getDataset(DatasetId datasetInstanceId, Map<String, String> arguments, @Nullable ClassLoader classLoader) throws DatasetManagementException, IOException {
            T t = super.getDataset(datasetInstanceId, arguments, classLoader);
            if (t instanceof UsageDataset) {
                @SuppressWarnings("unchecked") T t1 = (T) new WrappedUsageDataset((UsageDataset) t);
                return t1;
            }
            return t;
        }
    });
    // register usage for a stream and a dataset for single and multiple "owners", including a non-program
    registry.register(flow11, datasetInstance1);
    registry.register(flow12, stream1);
    registry.registerAll(ImmutableList.of(flow21, flow22), datasetInstance2);
    registry.registerAll(ImmutableList.of(flow21, flow22), stream1);
    int count = WrappedUsageDataset.registerCount;
    // validate usage
    Assert.assertEquals(ImmutableSet.of(datasetInstance1), registry.getDatasets(flow11));
    Assert.assertEquals(ImmutableSet.of(stream1), registry.getStreams(flow12));
    Assert.assertEquals(ImmutableSet.of(datasetInstance2), registry.getDatasets(flow21));
    Assert.assertEquals(ImmutableSet.of(datasetInstance2), registry.getDatasets(flow22));
    Assert.assertEquals(ImmutableSet.of(stream1), registry.getStreams(flow21));
    Assert.assertEquals(ImmutableSet.of(stream1), registry.getStreams(flow22));
    Assert.assertEquals(ImmutableSet.of(flow11), registry.getPrograms(datasetInstance1));
    Assert.assertEquals(ImmutableSet.of(flow21, flow22), registry.getPrograms(datasetInstance2));
    Assert.assertEquals(ImmutableSet.of(flow12, flow21, flow22), registry.getPrograms(stream1));
    // register datasets again
    registry.register(flow11, datasetInstance1);
    registry.registerAll(ImmutableList.of(flow21, flow22), datasetInstance2);
    // validate that this does re-register previous usages (DefaultUsageRegistry no longer avoids re-registration)
    count += 3;
    Assert.assertEquals(count, WrappedUsageDataset.registerCount);
    // validate usage
    Assert.assertEquals(ImmutableSet.of(datasetInstance1), registry.getDatasets(flow11));
    Assert.assertEquals(ImmutableSet.of(stream1), registry.getStreams(flow12));
    Assert.assertEquals(ImmutableSet.of(datasetInstance2), registry.getDatasets(flow21));
    Assert.assertEquals(ImmutableSet.of(datasetInstance2), registry.getDatasets(flow22));
    Assert.assertEquals(ImmutableSet.of(stream1), registry.getStreams(flow21));
    Assert.assertEquals(ImmutableSet.of(stream1), registry.getStreams(flow22));
    Assert.assertEquals(ImmutableSet.of(flow11), registry.getPrograms(datasetInstance1));
    Assert.assertEquals(ImmutableSet.of(flow21, flow22), registry.getPrograms(datasetInstance2));
    Assert.assertEquals(ImmutableSet.of(flow12, flow21, flow22), registry.getPrograms(stream1));
    // unregister app
    registry.unregister(flow11.getParent());
    // validate usage for that app is gone
    Assert.assertEquals(ImmutableSet.of(), registry.getDatasets(flow11));
    Assert.assertEquals(ImmutableSet.of(), registry.getStreams(flow12));
    Assert.assertEquals(ImmutableSet.of(datasetInstance2), registry.getDatasets(flow21));
    Assert.assertEquals(ImmutableSet.of(datasetInstance2), registry.getDatasets(flow22));
    Assert.assertEquals(ImmutableSet.of(stream1), registry.getStreams(flow21));
    Assert.assertEquals(ImmutableSet.of(stream1), registry.getStreams(flow22));
    Assert.assertEquals(ImmutableSet.of(), registry.getPrograms(datasetInstance1));
    Assert.assertEquals(ImmutableSet.of(flow21, flow22), registry.getPrograms(datasetInstance2));
    Assert.assertEquals(ImmutableSet.of(flow21, flow22), registry.getPrograms(stream1));
    // register application 1 again
    registry.register(flow11, datasetInstance1);
    registry.register(flow12, stream1);
    // validate it was re-registered
    Assert.assertEquals(ImmutableSet.of(datasetInstance1), registry.getDatasets(flow11));
    Assert.assertEquals(ImmutableSet.of(stream1), registry.getStreams(flow12));
    Assert.assertEquals(ImmutableSet.of(datasetInstance2), registry.getDatasets(flow21));
    Assert.assertEquals(ImmutableSet.of(datasetInstance2), registry.getDatasets(flow22));
    Assert.assertEquals(ImmutableSet.of(stream1), registry.getStreams(flow21));
    Assert.assertEquals(ImmutableSet.of(stream1), registry.getStreams(flow22));
    Assert.assertEquals(ImmutableSet.of(flow11), registry.getPrograms(datasetInstance1));
    Assert.assertEquals(ImmutableSet.of(flow21, flow22), registry.getPrograms(datasetInstance2));
    Assert.assertEquals(ImmutableSet.of(flow12, flow21, flow22), registry.getPrograms(stream1));
    // validate that this actually re-registered previous usages (through code in wrapped usage dataset)
    Assert.assertEquals(count + 2, WrappedUsageDataset.registerCount);
}

Example 85 with DatasetId

use of co.cask.cdap.proto.id.DatasetId in project cdap by caskdata.

the class SchemaIndexerTest method testSimpleSchema.

@Test
public void testSimpleSchema() throws Exception {
    Schema simpleSchema = Schema.of(Schema.Type.INT);
    Set<String> expected = Collections.emptySet();
    SchemaIndexer indexer = new SchemaIndexer();
    DatasetId datasetInstance = new DatasetId("ns1", "ds1");
    Set<String> actual = indexer.getIndexes(new MetadataEntry(datasetInstance, KEY, simpleSchema.toString()));
    Assert.assertEquals(expected, actual);
}