Examples with StreamId co.cask.cdap.proto.id.StreamId used on opensource projects

Example 81 with StreamId

use of co.cask.cdap.proto.id.StreamId in project cdap by caskdata.

the class StreamFileJanitorTestBase method testCleanupGeneration.

@Test
public void testCleanupGeneration() throws Exception {
    // Create a stream and performs couple truncate
    String streamName = "testCleanupGeneration";
    StreamId streamId = NamespaceId.DEFAULT.stream(streamName);
    StreamAdmin streamAdmin = getStreamAdmin();
    streamAdmin.create(streamId);
    StreamConfig streamConfig = streamAdmin.getConfig(streamId);
    StreamFileJanitor janitor = new StreamFileJanitor(getCConfiguration(), getStreamAdmin(), getNamespacedLocationFactory(), getNamespaceAdmin(), impersonator);
    for (int i = 0; i < 5; i++) {
        FileWriter<StreamEvent> writer = createWriter(streamId);
        writer.append(StreamFileTestUtils.createEvent(System.currentTimeMillis(), "Testing"));
        writer.close();
        // Call cleanup before truncate. The current generation should stand.
        janitor.clean(streamConfig.getLocation(), streamConfig.getTTL(), System.currentTimeMillis());
        verifyGeneration(streamConfig, i);
        streamAdmin.truncate(streamId);
    }
    int generation = StreamUtils.getGeneration(streamConfig);
    Assert.assertEquals(5, generation);
    janitor.clean(streamConfig.getLocation(), streamConfig.getTTL(), System.currentTimeMillis());
    // Verify the stream directory should only contains the generation directory
    for (Location location : streamConfig.getLocation().list()) {
        if (location.isDirectory()) {
            Assert.assertEquals(generation, Integer.parseInt(location.getName()));
        }
    }
}

Example 82 with StreamId

use of co.cask.cdap.proto.id.StreamId in project cdap by caskdata.

the class EntityIdKeyHelper method getTargetIdIdFromKey.

public static NamespacedEntityId getTargetIdIdFromKey(MDSKey.Splitter keySplitter, String type) {
    if (type.equals(TYPE_MAP.get(NamespaceId.class))) {
        String namespaceId = keySplitter.getString();
        return new NamespaceId(namespaceId);
    } else if (type.equals(TYPE_MAP.get(ProgramId.class))) {
        String namespaceId = keySplitter.getString();
        String appId = keySplitter.getString();
        String programType = keySplitter.getString();
        String programId = keySplitter.getString();
        return new ProgramId(namespaceId, appId, programType, programId);
    } else if (type.equals(TYPE_MAP.get(ApplicationId.class))) {
        String namespaceId = keySplitter.getString();
        String appId = keySplitter.getString();
        return new ApplicationId(namespaceId, appId);
    } else if (type.equals(TYPE_MAP.get(ArtifactId.class))) {
        String namespaceId = keySplitter.getString();
        String name = keySplitter.getString();
        String version = keySplitter.getString();
        return new ArtifactId(namespaceId, name, version);
    } else if (type.equals(TYPE_MAP.get(DatasetId.class))) {
        String namespaceId = keySplitter.getString();
        String instanceId = keySplitter.getString();
        return new DatasetId(namespaceId, instanceId);
    } else if (type.equals(TYPE_MAP.get(StreamId.class))) {
        String namespaceId = keySplitter.getString();
        String instanceId = keySplitter.getString();
        return new StreamId(namespaceId, instanceId);
    } else if (type.equals(TYPE_MAP.get(StreamViewId.class))) {
        String namespaceId = keySplitter.getString();
        String streamId = keySplitter.getString();
        String viewId = keySplitter.getString();
        return new StreamViewId(namespaceId, streamId, viewId);
    }
    throw new IllegalArgumentException("Illegal Type " + type + " of metadata source.");
}

Example 83 with StreamId

use of co.cask.cdap.proto.id.StreamId in project cdap by caskdata.

the class FileStreamAdmin method deleteView.

@Override
public void deleteView(final StreamViewId viewId) throws Exception {
    final StreamId stream = viewId.getParent();
    streamCoordinatorClient.exclusiveAction(stream, new Callable<Void>() {

        @Override
        public Void call() throws Exception {
            if (!exists(stream)) {
                throw new StreamNotFoundException(stream);
            }
            viewAdmin.delete(viewId);
            return null;
        }
    });
}

Example 84 with StreamId

use of co.cask.cdap.proto.id.StreamId in project cdap by caskdata.

the class InMemoryStreamAdmin method dropAllInNamespace.

@Override
public void dropAllInNamespace(NamespaceId namespace) throws Exception {
    queueService.resetStreamsWithPrefix(QueueName.prefixForNamedspacedStream(namespace.getNamespace()));
    for (StreamSpecification spec : streamMetaStore.listStreams(namespace)) {
        // Remove metadata for the stream
        StreamId stream = namespace.stream(spec.getName());
        metadataStore.removeMetadata(stream);
        streamMetaStore.removeStream(stream);
    }
}

Example 85 with StreamId

use of co.cask.cdap.proto.id.StreamId in project cdap by caskdata.

the class UsageDatasetTest method testAllMappings.

@Test
public void testAllMappings() throws Exception {
    final UsageDataset usageDataset = getUsageDataset("testAllMappings");
    TransactionExecutor txnl = dsFrameworkUtil.newInMemoryTransactionExecutor((TransactionAware) usageDataset);
    // Add mappings
    txnl.execute(new TransactionExecutor.Subroutine() {

        @Override
        public void apply() throws Exception {
            usageDataset.register(flow11, datasetInstance1);
            usageDataset.register(service21, datasetInstance3);
            usageDataset.register(flow12, stream1);
        }
    });
    txnl.execute(new TransactionExecutor.Subroutine() {

        @Override
        public void apply() throws Exception {
            // Verify app mappings
            Assert.assertEquals(ImmutableSet.of(datasetInstance1), usageDataset.getDatasets(flow11));
            Assert.assertEquals(ImmutableSet.<DatasetId>of(), usageDataset.getDatasets(flow12));
            Assert.assertEquals(ImmutableSet.of(datasetInstance3), usageDataset.getDatasets(service21));
            Assert.assertEquals(ImmutableSet.of(stream1), usageDataset.getStreams(flow12));
            Assert.assertEquals(ImmutableSet.<StreamId>of(), usageDataset.getStreams(flow22));
            // Verify dataset/stream mappings
            Assert.assertEquals(ImmutableSet.of(flow11), usageDataset.getPrograms(datasetInstance1));
            Assert.assertEquals(ImmutableSet.of(flow12), usageDataset.getPrograms(stream1));
            Assert.assertEquals(ImmutableSet.<ProgramId>of(), usageDataset.getPrograms(stream2));
        }
    });
    // --------- Delete app1 -----------
    txnl.execute(new TransactionExecutor.Subroutine() {

        @Override
        public void apply() throws Exception {
            usageDataset.unregister(flow11.getParent());
        }
    });
    txnl.execute(new TransactionExecutor.Subroutine() {

        @Override
        public void apply() throws Exception {
            // Verify app mappings
            Assert.assertEquals(ImmutableSet.<DatasetId>of(), usageDataset.getDatasets(flow11));
            Assert.assertEquals(ImmutableSet.<DatasetId>of(), usageDataset.getDatasets(flow12));
            Assert.assertEquals(ImmutableSet.of(datasetInstance3), usageDataset.getDatasets(service21));
            Assert.assertEquals(ImmutableSet.<StreamId>of(), usageDataset.getStreams(flow12));
            Assert.assertEquals(ImmutableSet.<StreamId>of(), usageDataset.getStreams(flow22));
            // Verify dataset/stream mappings
            Assert.assertEquals(ImmutableSet.<ProgramId>of(), usageDataset.getPrograms(datasetInstance1));
            Assert.assertEquals(ImmutableSet.<ProgramId>of(), usageDataset.getPrograms(stream1));
            Assert.assertEquals(ImmutableSet.<ProgramId>of(), usageDataset.getPrograms(stream2));
            // Verify app mappings
            Assert.assertEquals(ImmutableSet.<DatasetId>of(), usageDataset.getDatasets(flow11));
            Assert.assertEquals(ImmutableSet.<DatasetId>of(), usageDataset.getDatasets(flow12));
            Assert.assertEquals(ImmutableSet.of(datasetInstance3), usageDataset.getDatasets(service21));
            Assert.assertEquals(ImmutableSet.<StreamId>of(), usageDataset.getStreams(flow12));
            Assert.assertEquals(ImmutableSet.<StreamId>of(), usageDataset.getStreams(flow22));
            // Verify dataset/stream mappings
            Assert.assertEquals(ImmutableSet.<ProgramId>of(), usageDataset.getPrograms(datasetInstance1));
            Assert.assertEquals(ImmutableSet.<ProgramId>of(), usageDataset.getPrograms(stream1));
            Assert.assertEquals(ImmutableSet.<ProgramId>of(), usageDataset.getPrograms(stream2));
            // Verify app mappings
            Assert.assertEquals(ImmutableSet.<DatasetId>of(), usageDataset.getDatasets(flow11));
            Assert.assertEquals(ImmutableSet.<DatasetId>of(), usageDataset.getDatasets(flow12));
            Assert.assertEquals(ImmutableSet.of(datasetInstance3), usageDataset.getDatasets(service21));
            Assert.assertEquals(ImmutableSet.<StreamId>of(), usageDataset.getStreams(flow12));
            Assert.assertEquals(ImmutableSet.<StreamId>of(), usageDataset.getStreams(flow22));
            // Verify dataset/stream mappings
            Assert.assertEquals(ImmutableSet.<ProgramId>of(), usageDataset.getPrograms(datasetInstance1));
            Assert.assertEquals(ImmutableSet.<ProgramId>of(), usageDataset.getPrograms(stream1));
            Assert.assertEquals(ImmutableSet.<ProgramId>of(), usageDataset.getPrograms(stream2));
        }
    });
    // --------- Delete app2 -----------
    txnl.execute(new TransactionExecutor.Subroutine() {

        @Override
        public void apply() throws Exception {
            usageDataset.unregister(flow21.getParent());
        }
    });
    txnl.execute(new TransactionExecutor.Subroutine() {

        @Override
        public void apply() throws Exception {
            // Verify app mappings
            Assert.assertEquals(ImmutableSet.<DatasetId>of(), usageDataset.getDatasets(flow11));
            Assert.assertEquals(ImmutableSet.<DatasetId>of(), usageDataset.getDatasets(flow12));
            Assert.assertEquals(ImmutableSet.<DatasetId>of(), usageDataset.getDatasets(service21));
            Assert.assertEquals(ImmutableSet.<StreamId>of(), usageDataset.getStreams(flow12));
            Assert.assertEquals(ImmutableSet.<StreamId>of(), usageDataset.getStreams(flow22));
            // Verify dataset/stream mappings
            Assert.assertEquals(ImmutableSet.<ProgramId>of(), usageDataset.getPrograms(datasetInstance1));
            Assert.assertEquals(ImmutableSet.<ProgramId>of(), usageDataset.getPrograms(stream1));
            Assert.assertEquals(ImmutableSet.<ProgramId>of(), usageDataset.getPrograms(stream2));
        }
    });
}