Examples with StreamId co.cask.cdap.proto.id.StreamId used on opensource projects

Example 26 with StreamId

use of co.cask.cdap.proto.id.StreamId in project cdap by caskdata.

the class StreamFileSizeFetcherTest method testFetchSize.

@Test
public void testFetchSize() throws Exception {
    final String streamName = "testFetchSize";
    StreamId streamId = NamespaceId.DEFAULT.stream(streamName);
    final int nbEvents = 100;
    StreamAdmin streamAdmin = new TestStreamAdmin(namespacedLocationFactory, Long.MAX_VALUE, 1000);
    streamAdmin.create(streamId);
    StreamConfig config = streamAdmin.getConfig(streamId);
    try {
        StreamUtils.fetchStreamFilesSize(StreamUtils.createGenerationLocation(config.getLocation(), StreamUtils.getGeneration(config)));
        Assert.fail("No stream file created yet");
    } catch (IOException e) {
    // Expected
    }
    // Creates a stream file that has no event inside
    Location partitionLocation = StreamUtils.createPartitionLocation(config.getLocation(), 0, Long.MAX_VALUE);
    Location dataLocation = StreamUtils.createStreamLocation(partitionLocation, "writer", 0, StreamFileType.EVENT);
    Location idxLocation = StreamUtils.createStreamLocation(partitionLocation, "writer", 0, StreamFileType.INDEX);
    StreamDataFileWriter writer = new StreamDataFileWriter(Locations.newOutputSupplier(dataLocation), Locations.newOutputSupplier(idxLocation), 10000L);
    // Write 100 events to the stream
    for (int i = 0; i < nbEvents; i++) {
        writer.append(StreamFileTestUtils.createEvent(i, "foo"));
    }
    writer.close();
    long size = StreamUtils.fetchStreamFilesSize(StreamUtils.createGenerationLocation(config.getLocation(), StreamUtils.getGeneration(config)));
    Assert.assertTrue(size > 0);
    Assert.assertEquals(dataLocation.length(), size);
}

Example 27 with StreamId

use of co.cask.cdap.proto.id.StreamId in project cdap by caskdata.

the class StreamFileJanitorTestBase method testCleanupGeneration.

@Test
public void testCleanupGeneration() throws Exception {
    // Create a stream and performs couple truncate
    String streamName = "testCleanupGeneration";
    StreamId streamId = NamespaceId.DEFAULT.stream(streamName);
    StreamAdmin streamAdmin = getStreamAdmin();
    streamAdmin.create(streamId);
    StreamConfig streamConfig = streamAdmin.getConfig(streamId);
    StreamFileJanitor janitor = new StreamFileJanitor(getCConfiguration(), getStreamAdmin(), getNamespacedLocationFactory(), getNamespaceAdmin(), impersonator);
    for (int i = 0; i < 5; i++) {
        FileWriter<StreamEvent> writer = createWriter(streamId);
        writer.append(StreamFileTestUtils.createEvent(System.currentTimeMillis(), "Testing"));
        writer.close();
        // Call cleanup before truncate. The current generation should stand.
        janitor.clean(streamConfig.getLocation(), streamConfig.getTTL(), System.currentTimeMillis());
        verifyGeneration(streamConfig, i);
        streamAdmin.truncate(streamId);
    }
    int generation = StreamUtils.getGeneration(streamConfig);
    Assert.assertEquals(5, generation);
    janitor.clean(streamConfig.getLocation(), streamConfig.getTTL(), System.currentTimeMillis());
    // Verify the stream directory should only contains the generation directory
    for (Location location : streamConfig.getLocation().list()) {
        if (location.isDirectory()) {
            Assert.assertEquals(generation, Integer.parseInt(location.getName()));
        }
    }
}

Example 28 with StreamId

use of co.cask.cdap.proto.id.StreamId in project cdap by caskdata.

the class EntityIdKeyHelper method getTargetIdIdFromKey.

public static NamespacedEntityId getTargetIdIdFromKey(MDSKey.Splitter keySplitter, String type) {
    if (type.equals(TYPE_MAP.get(NamespaceId.class))) {
        String namespaceId = keySplitter.getString();
        return new NamespaceId(namespaceId);
    } else if (type.equals(TYPE_MAP.get(ProgramId.class))) {
        String namespaceId = keySplitter.getString();
        String appId = keySplitter.getString();
        String programType = keySplitter.getString();
        String programId = keySplitter.getString();
        return new ProgramId(namespaceId, appId, programType, programId);
    } else if (type.equals(TYPE_MAP.get(ApplicationId.class))) {
        String namespaceId = keySplitter.getString();
        String appId = keySplitter.getString();
        return new ApplicationId(namespaceId, appId);
    } else if (type.equals(TYPE_MAP.get(ArtifactId.class))) {
        String namespaceId = keySplitter.getString();
        String name = keySplitter.getString();
        String version = keySplitter.getString();
        return new ArtifactId(namespaceId, name, version);
    } else if (type.equals(TYPE_MAP.get(DatasetId.class))) {
        String namespaceId = keySplitter.getString();
        String instanceId = keySplitter.getString();
        return new DatasetId(namespaceId, instanceId);
    } else if (type.equals(TYPE_MAP.get(StreamId.class))) {
        String namespaceId = keySplitter.getString();
        String instanceId = keySplitter.getString();
        return new StreamId(namespaceId, instanceId);
    } else if (type.equals(TYPE_MAP.get(StreamViewId.class))) {
        String namespaceId = keySplitter.getString();
        String streamId = keySplitter.getString();
        String viewId = keySplitter.getString();
        return new StreamViewId(namespaceId, streamId, viewId);
    }
    throw new IllegalArgumentException("Illegal Type " + type + " of metadata source.");
}

Example 29 with StreamId

use of co.cask.cdap.proto.id.StreamId in project cdap by caskdata.

the class FileStreamAdmin method deleteView.

@Override
public void deleteView(final StreamViewId viewId) throws Exception {
    final StreamId stream = viewId.getParent();
    streamCoordinatorClient.exclusiveAction(stream, new Callable<Void>() {

        @Override
        public Void call() throws Exception {
            if (!exists(stream)) {
                throw new StreamNotFoundException(stream);
            }
            viewAdmin.delete(viewId);
            return null;
        }
    });
}

Example 30 with StreamId

use of co.cask.cdap.proto.id.StreamId in project cdap by caskdata.

the class InMemoryStreamAdmin method dropAllInNamespace.

@Override
public void dropAllInNamespace(NamespaceId namespace) throws Exception {
    queueService.resetStreamsWithPrefix(QueueName.prefixForNamedspacedStream(namespace.getNamespace()));
    for (StreamSpecification spec : streamMetaStore.listStreams(namespace)) {
        // Remove metadata for the stream
        StreamId stream = namespace.stream(spec.getName());
        metadataStore.removeMetadata(stream);
        streamMetaStore.removeStream(stream);
    }
}