Examples with StreamId co.cask.cdap.proto.id.StreamId used on opensource projects

Example 71 with StreamId

use of co.cask.cdap.proto.id.StreamId in project cdap by caskdata.

the class MetadataHttpHandlerTestRun method testSearchMetadataDelete.

@Test
public void testSearchMetadataDelete() throws Exception {
    NamespaceId namespace = new NamespaceId("ns1");
    namespaceClient.create(new NamespaceMeta.Builder().setName(namespace).build());
    // Deploy app
    appClient.deploy(namespace, createAppJarFile(WordCountApp.class, WordCountApp.class.getSimpleName(), "1.0"));
    Set<String> tags = ImmutableSet.of("tag1", "tag2");
    ArtifactId artifact = namespace.artifact("WordCountApp", "1.0");
    ApplicationId app = namespace.app("WordCountApp");
    ProgramId flow = app.flow("WordCountFlow");
    ProgramId service = app.service("WordFrequencyService");
    StreamId stream = namespace.stream("text");
    DatasetId datasetInstance = namespace.dataset("mydataset");
    StreamViewId view = stream.view("view");
    streamViewClient.createOrUpdate(view, new ViewSpecification(new FormatSpecification("csv", null, null)));
    // Add metadata
    addTags(app, tags);
    addTags(flow, tags);
    addTags(stream, tags);
    addTags(datasetInstance, tags);
    addTags(view, tags);
    // Assert metadata
    Assert.assertEquals(ImmutableSet.of(new MetadataSearchResultRecord(stream), new MetadataSearchResultRecord(view)), searchMetadata(namespace, "text"));
    Assert.assertEquals(ImmutableSet.of(new MetadataSearchResultRecord(datasetInstance)), searchMetadata(namespace, "mydataset"));
    Assert.assertEquals(ImmutableSet.of(new MetadataSearchResultRecord(app), new MetadataSearchResultRecord(flow), new MetadataSearchResultRecord(artifact), new MetadataSearchResultRecord(service)), searchMetadata(namespace, "word*"));
    Assert.assertEquals(ImmutableSet.of(new MetadataSearchResultRecord(app), new MetadataSearchResultRecord(flow), new MetadataSearchResultRecord(stream), new MetadataSearchResultRecord(datasetInstance), new MetadataSearchResultRecord(view)), searchMetadata(namespace, "tag1"));
    // Delete entities
    appClient.delete(app);
    streamViewClient.delete(view);
    streamClient.delete(stream);
    datasetClient.delete(datasetInstance);
    artifactClient.delete(artifact);
    // Assert no metadata
    Assert.assertEquals(ImmutableSet.of(), searchMetadata(namespace, "text"));
    Assert.assertEquals(ImmutableSet.of(), searchMetadata(namespace, "mydataset"));
    Assert.assertEquals(ImmutableSet.of(), searchMetadata(namespace, "word*"));
    Assert.assertEquals(ImmutableSet.of(), searchMetadata(namespace, "tag1"));
}

Example 72 with StreamId

use of co.cask.cdap.proto.id.StreamId in project cdap by caskdata.

the class ConcurrentStreamWriterTestBase method testConcurrentAppendFile.

@Test
public void testConcurrentAppendFile() throws Exception {
    final String streamName = "testConcurrentFile";
    NamespaceId namespace = new NamespaceId("namespace");
    StreamId streamId = namespace.stream(streamName);
    StreamAdmin streamAdmin = new TestStreamAdmin(getNamespacedLocationFactory(), Long.MAX_VALUE, 1000);
    int threads = Runtime.getRuntime().availableProcessors() * 4;
    StreamFileWriterFactory fileWriterFactory = createStreamFileWriterFactory();
    final ConcurrentStreamWriter streamWriter = createStreamWriter(streamId, streamAdmin, threads, fileWriterFactory);
    int msgCount = 10000;
    NamespacedLocationFactory locationFactory = getNamespacedLocationFactory();
    // Half of the threads will be calling appendFile, then other half append event one by one
    // Prepare the files first, each file has 10000 events.
    final List<FileInfo> fileInfos = Lists.newArrayList();
    for (int i = 0; i < threads / 2; i++) {
        fileInfos.add(generateFile(locationFactory, i, msgCount));
    }
    // Append file and write events
    final CountDownLatch startLatch = new CountDownLatch(1);
    final CountDownLatch completion = new CountDownLatch(threads);
    ExecutorService executor = Executors.newFixedThreadPool(threads);
    for (int i = 0; i < threads / 2; i++) {
        executor.execute(createAppendFileTask(streamId, streamWriter, fileInfos.get(i), startLatch, completion));
    }
    for (int i = threads / 2; i < threads; i++) {
        executor.execute(createWriterTask(streamId, streamWriter, i, msgCount, 50, startLatch, completion));
    }
    startLatch.countDown();
    Assert.assertTrue(completion.await(4, TimeUnit.MINUTES));
    // Verify all events are written.
    // There should be only one partition
    Location partitionLocation = streamAdmin.getConfig(streamId).getLocation().list().get(0);
    List<Location> files = partitionLocation.list();
    List<StreamEvent> events = Lists.newArrayListWithCapacity(threads * msgCount);
    for (Location location : files) {
        // Only create reader for the event file
        if (StreamFileType.getType(location.getName()) != StreamFileType.EVENT) {
            continue;
        }
        StreamDataFileReader reader = StreamDataFileReader.create(Locations.newInputSupplier(location));
        reader.read(events, Integer.MAX_VALUE, 0, TimeUnit.SECONDS);
    }
    Assert.assertTrue(verifyEvents(threads, msgCount, events));
}

Example 73 with StreamId

use of co.cask.cdap.proto.id.StreamId in project cdap by caskdata.

the class StreamAdminTest method testCreateExist.

@Test
public void testCreateExist() throws Exception {
    SecurityRequestContext.setUserId(USER.getName());
    StreamAdmin streamAdmin = getStreamAdmin();
    String streamName = "streamName";
    StreamId streamId = FOO_NAMESPACE.stream(streamName);
    StreamId otherStreamId = OTHER_NAMESPACE.stream(streamName);
    Assert.assertFalse(streamAdmin.exists(streamId));
    Assert.assertFalse(streamAdmin.exists(otherStreamId));
    try {
        streamAdmin.create(streamId);
        Assert.fail("User should not be able to create a stream in this namespace.");
    } catch (UnauthorizedException e) {
    // expected
    }
    // grant write access for user to foo_namespace
    grantAndAssertSuccess(streamId.getParent(), USER, ImmutableSet.of(Action.WRITE));
    streamAdmin.create(streamId);
    // Even though both streams have the same name, {@code otherStreamId} does not exist because it is in a different
    // namespace than the one created above.
    Assert.assertTrue(streamAdmin.exists(streamId));
    Assert.assertFalse(streamAdmin.exists(otherStreamId));
    try {
        streamAdmin.create(otherStreamId);
        Assert.fail("User should not be able to create a stream in this namespace.");
    } catch (UnauthorizedException e) {
    // expected
    }
    // grant write access for user to other_namespace
    grantAndAssertSuccess(otherStreamId.getParent(), USER, ImmutableSet.of(Action.WRITE));
    streamAdmin.create(otherStreamId);
    Assert.assertTrue(streamAdmin.exists(otherStreamId));
    // the user should be able to drop the stream, they had created
    streamAdmin.drop(otherStreamId);
    Assert.assertFalse(streamAdmin.exists(otherStreamId));
    // revoke the permission for the user on the stream in foo_namespace
    revokeAndAssertSuccess(streamId, USER, EnumSet.allOf(Action.class));
    try {
        streamAdmin.drop(streamId);
        Assert.fail("User should not be able to delete a stream in this namespace.");
    } catch (UnauthorizedException e) {
    // expected
    }
    // grant WRITE permission to the user but they still should not be able to drop the stream
    grantAndAssertSuccess(streamId, USER, ImmutableSet.of(Action.WRITE));
    try {
        streamAdmin.drop(streamId);
        Assert.fail("User should not be able to delete a stream with only Write Action access.");
    } catch (UnauthorizedException e) {
    // expected
    }
    // grant admin access and the user should then be able to drop the stream
    grantAndAssertSuccess(streamId, USER, ImmutableSet.of(Action.ADMIN));
    streamAdmin.drop(streamId);
    Assert.assertFalse(streamAdmin.exists(streamId));
}

Example 74 with StreamId

use of co.cask.cdap.proto.id.StreamId in project cdap by caskdata.

the class StreamAdminTest method testDropAllInNamespace.

@Test
public void testDropAllInNamespace() throws Exception {
    StreamAdmin streamAdmin = getStreamAdmin();
    grantAndAssertSuccess(FOO_NAMESPACE, USER, ImmutableSet.of(Action.WRITE, Action.ADMIN));
    grantAndAssertSuccess(OTHER_NAMESPACE, USER, ImmutableSet.of(Action.WRITE, Action.ADMIN));
    StreamId otherStream = OTHER_NAMESPACE.stream("otherStream");
    List<StreamId> fooStreams = Lists.newArrayList();
    for (int i = 0; i < 4; i++) {
        fooStreams.add(FOO_NAMESPACE.stream("stream" + i));
    }
    List<StreamId> allStreams = Lists.newArrayList();
    allStreams.addAll(fooStreams);
    allStreams.add(otherStream);
    for (StreamId stream : allStreams) {
        streamAdmin.create(stream);
        writeEvent(stream);
        // all of the streams should have data in it after writing to them
        Assert.assertNotEquals(0, getStreamSize(stream));
    }
    streamAdmin.dropAllInNamespace(FOO_NAMESPACE);
    // All of the streams within the default namespace should no longer exist
    for (StreamId defaultStream : fooStreams) {
        Assert.assertFalse(streamAdmin.exists(defaultStream));
    }
    // otherStream isn't in the foo namespace so its data is not deleted in the above call to dropAllInNamespace.
    Assert.assertNotEquals(0, getStreamSize(otherStream));
    // truncate should also delete all the data of a stream
    streamAdmin.truncate(otherStream);
    Assert.assertEquals(0, getStreamSize(otherStream));
}

Example 75 with StreamId

use of co.cask.cdap.proto.id.StreamId in project cdap by caskdata.

the class StreamAdminTest method testAuditPublish.

@Test
public void testAuditPublish() throws Exception {
    grantAndAssertSuccess(FOO_NAMESPACE, USER, EnumSet.allOf(Action.class));
    // clear existing all messages
    getInMemoryAuditPublisher().popMessages();
    final List<AuditMessage> expectedMessages = new ArrayList<>();
    StreamAdmin streamAdmin = getStreamAdmin();
    StreamId stream1 = FOO_NAMESPACE.stream("stream1");
    streamAdmin.create(stream1);
    expectedMessages.add(new AuditMessage(0, stream1, "", AuditType.CREATE, AuditPayload.EMPTY_PAYLOAD));
    StreamId stream2 = FOO_NAMESPACE.stream("stream2");
    streamAdmin.create(stream2);
    expectedMessages.add(new AuditMessage(0, stream2, "", AuditType.CREATE, AuditPayload.EMPTY_PAYLOAD));
    streamAdmin.truncate(stream1);
    expectedMessages.add(new AuditMessage(0, stream1, "", AuditType.TRUNCATE, AuditPayload.EMPTY_PAYLOAD));
    streamAdmin.updateConfig(stream1, new StreamProperties(100L, new FormatSpecification("f", null), 100));
    expectedMessages.add(new AuditMessage(0, stream1, "", AuditType.UPDATE, AuditPayload.EMPTY_PAYLOAD));
    ProgramRunId run = new ProgramId("ns1", "app", ProgramType.FLOW, "flw").run(RunIds.generate().getId());
    streamAdmin.addAccess(run, stream1, AccessType.READ);
    expectedMessages.add(new AuditMessage(0, stream1, "", AuditType.ACCESS, new AccessPayload(co.cask.cdap.proto.audit.payload.access.AccessType.READ, run)));
    streamAdmin.drop(stream1);
    expectedMessages.add(new AuditMessage(0, stream1, "", AuditType.DELETE, AuditPayload.EMPTY_PAYLOAD));
    streamAdmin.dropAllInNamespace(FOO_NAMESPACE);
    expectedMessages.add(new AuditMessage(0, stream2, "", AuditType.DELETE, AuditPayload.EMPTY_PAYLOAD));
    // Ignore audit messages for system namespace (creation of system datasets, etc)
    final String systemNs = NamespaceId.SYSTEM.getNamespace();
    final Iterable<AuditMessage> actualMessages = Iterables.filter(getInMemoryAuditPublisher().popMessages(), new Predicate<AuditMessage>() {

        @Override
        public boolean apply(AuditMessage input) {
            return !(input.getEntityId() instanceof NamespacedEntityId && ((NamespacedEntityId) input.getEntityId()).getNamespace().equals(systemNs));
        }
    });
    Assert.assertEquals(expectedMessages, Lists.newArrayList(actualMessages));
}