Examples with StreamId co.cask.cdap.proto.id.StreamId used on opensource projects

Example 21 with StreamId

use of co.cask.cdap.proto.id.StreamId in project cdap by caskdata.

the class StreamAdminTest method testAuditPublish.

@Test
public void testAuditPublish() throws Exception {
    grantAndAssertSuccess(FOO_NAMESPACE, USER, EnumSet.allOf(Action.class));
    // clear existing all messages
    getInMemoryAuditPublisher().popMessages();
    final List<AuditMessage> expectedMessages = new ArrayList<>();
    StreamAdmin streamAdmin = getStreamAdmin();
    StreamId stream1 = FOO_NAMESPACE.stream("stream1");
    streamAdmin.create(stream1);
    expectedMessages.add(new AuditMessage(0, stream1, "", AuditType.CREATE, AuditPayload.EMPTY_PAYLOAD));
    StreamId stream2 = FOO_NAMESPACE.stream("stream2");
    streamAdmin.create(stream2);
    expectedMessages.add(new AuditMessage(0, stream2, "", AuditType.CREATE, AuditPayload.EMPTY_PAYLOAD));
    streamAdmin.truncate(stream1);
    expectedMessages.add(new AuditMessage(0, stream1, "", AuditType.TRUNCATE, AuditPayload.EMPTY_PAYLOAD));
    streamAdmin.updateConfig(stream1, new StreamProperties(100L, new FormatSpecification("f", null), 100));
    expectedMessages.add(new AuditMessage(0, stream1, "", AuditType.UPDATE, AuditPayload.EMPTY_PAYLOAD));
    ProgramRunId run = new ProgramId("ns1", "app", ProgramType.FLOW, "flw").run(RunIds.generate().getId());
    streamAdmin.addAccess(run, stream1, AccessType.READ);
    expectedMessages.add(new AuditMessage(0, stream1, "", AuditType.ACCESS, new AccessPayload(co.cask.cdap.proto.audit.payload.access.AccessType.READ, run)));
    streamAdmin.drop(stream1);
    expectedMessages.add(new AuditMessage(0, stream1, "", AuditType.DELETE, AuditPayload.EMPTY_PAYLOAD));
    streamAdmin.dropAllInNamespace(FOO_NAMESPACE);
    expectedMessages.add(new AuditMessage(0, stream2, "", AuditType.DELETE, AuditPayload.EMPTY_PAYLOAD));
    // Ignore audit messages for system namespace (creation of system datasets, etc)
    final String systemNs = NamespaceId.SYSTEM.getNamespace();
    final Iterable<AuditMessage> actualMessages = Iterables.filter(getInMemoryAuditPublisher().popMessages(), new Predicate<AuditMessage>() {

        @Override
        public boolean apply(AuditMessage input) {
            return !(input.getEntityId() instanceof NamespacedEntityId && ((NamespacedEntityId) input.getEntityId()).getNamespace().equals(systemNs));
        }
    });
    Assert.assertEquals(expectedMessages, Lists.newArrayList(actualMessages));
}

Example 22 with StreamId

use of co.cask.cdap.proto.id.StreamId in project cdap by caskdata.

the class StreamAdminTest method testOwner.

@Test
public void testOwner() throws Exception {
    // crate a stream with owner
    StreamAdmin streamAdmin = getStreamAdmin();
    OwnerAdmin ownerAdmin = getOwnerAdmin();
    grantAndAssertSuccess(FOO_NAMESPACE, USER, ImmutableSet.of(Action.WRITE));
    StreamId stream = FOO_NAMESPACE.stream("stream");
    Properties properties = new Properties();
    String ownerPrincipal = "user/somehost@somekdc.net";
    properties.put(Constants.Security.PRINCIPAL, ownerPrincipal);
    streamAdmin.create(stream, properties);
    Assert.assertTrue(streamAdmin.exists(stream));
    // Check that the owner information got stored in owner store
    Assert.assertTrue(ownerAdmin.exists(stream));
    // also verify that we are able to get owner information back in properties
    Assert.assertEquals(ownerPrincipal, streamAdmin.getProperties(stream).getOwnerPrincipal());
    // updating stream owner should fail
    try {
        streamAdmin.updateConfig(stream, new StreamProperties(1L, null, null, null, "user/somekdc.net"));
        Assert.fail();
    } catch (UnauthorizedException e) {
    // expected
    }
    // trying to create same stream with different owner should fail
    properties.put(Constants.Security.PRINCIPAL, "someOtherUser/someHost@somekdc.net");
    try {
        streamAdmin.create(stream, properties);
        Assert.fail("Should have failed to add the same stream with different owner");
    } catch (UnauthorizedException e) {
    // expected
    }
    // ensure that the previous owner still exists
    Assert.assertEquals(ownerPrincipal, streamAdmin.getProperties(stream).getOwnerPrincipal());
    // drop the stream which should also delete the owner info
    streamAdmin.drop(stream);
    Assert.assertFalse(ownerAdmin.exists(stream));
}

Example 23 with StreamId

use of co.cask.cdap.proto.id.StreamId in project cdap by caskdata.

the class StreamConsumerStateTestBase method testNamespacedStore.

@Test
public void testNamespacedStore() throws Exception {
    // Store different offsets for two streams using the same StateStoreFactory to show that
    // StateStoreFactory is capable of storing distinct states for streams with same name but different namespace
    StreamAdmin streamAdmin = getStreamAdmin();
    String streamName = "testNamespacedStore";
    StreamId streamId = TEST_NAMESPACE.stream(streamName);
    StreamId otherStreamId = OTHER_NAMESPACE.stream(streamName);
    streamAdmin.create(streamId);
    streamAdmin.create(otherStreamId);
    StreamConfig config = streamAdmin.getConfig(streamId);
    StreamConfig otherConfig = streamAdmin.getConfig(otherStreamId);
    // Creates a state with 4 offsets
    StreamConsumerState state = generateState(0L, 0, config, 0L, 4);
    StreamConsumerStateStore stateStore = createStateStore(config);
    // Create another state with more offsets for stream in different namespace
    StreamConsumerState otherState = generateState(0L, 0, otherConfig, 0L, 8);
    StreamConsumerStateStore otherStateStore = createStateStore(otherConfig);
    // Save the states.
    stateStore.save(state);
    otherStateStore.save(otherState);
    // Read the state back
    StreamConsumerState readState = stateStore.get(0, 0);
    StreamConsumerState otherReadState = otherStateStore.get(0, 0);
    Assert.assertEquals(state, readState);
    Assert.assertEquals(otherState, otherReadState);
    Assert.assertNotEquals(state, otherState);
}

Example 24 with StreamId

use of co.cask.cdap.proto.id.StreamId in project cdap by caskdata.

the class StreamConsumerStateTestBase method testRemove.

@Test
public void testRemove() throws Exception {
    StreamAdmin streamAdmin = getStreamAdmin();
    String streamName = "testRemove";
    StreamId streamId = TEST_NAMESPACE.stream(streamName);
    streamAdmin.create(streamId);
    StreamConfig config = streamAdmin.getConfig(streamId);
    // Creates 4 states of 2 groups, each with 4 offsets
    Set<StreamConsumerState> states = Sets.newHashSet();
    for (int i = 0; i < 4; i++) {
        states.add(generateState(i % 2, i, config, 0L, 4));
    }
    StreamConsumerStateStore stateStore = createStateStore(config);
    stateStore.save(states);
    // Read all states back
    Set<StreamConsumerState> readStates = Sets.newHashSet();
    stateStore.getAll(readStates);
    Assert.assertEquals(states, readStates);
    // Remove groupId 0
    Set<StreamConsumerState> removeStates = Sets.newHashSet();
    for (StreamConsumerState state : readStates) {
        if (state.getGroupId() == 0) {
            removeStates.add(state);
        }
    }
    stateStore.remove(removeStates);
    // Read all states back
    readStates.clear();
    stateStore.getAll(readStates);
    Assert.assertEquals(2, readStates.size());
    for (StreamConsumerState state : readStates) {
        Assert.assertEquals(1L, state.getGroupId());
    }
}

Example 25 with StreamId

use of co.cask.cdap.proto.id.StreamId in project cdap by caskdata.

the class TimePartitionedStreamTestBase method testTimePartition.

@Test
public void testTimePartition() throws IOException {
    // Create time partition file of 1 seconds each.
    String streamName = "stream";
    Location streamLocation = getLocationFactory().create(streamName);
    streamLocation.mkdirs();
    TimePartitionedStreamFileWriter writer = new TimePartitionedStreamFileWriter(streamLocation, 1000, "file", 100, new StreamId(NamespaceId.DEFAULT.getNamespace(), streamName), impersonator);
    // Write 2 events per millis for 3 seconds, starting at 0.5 second.
    long timeBase = 500;
    for (int i = 0; i < 3; i++) {
        for (int j = 0; j < 1000; j++) {
            long offset = i * 1000 + j;
            long timestamp = timeBase + offset;
            writer.append(StreamFileTestUtils.createEvent(timestamp, "Testing " + offset + " 0"));
            writer.append(StreamFileTestUtils.createEvent(timestamp, "Testing " + offset + " 1"));
        }
    }
    writer.close();
    // There should be four partition directory (500-1000, 1000-2000, 2000-3000, 3000-3500).
    List<Location> partitionDirs = Lists.newArrayList(streamLocation.list());
    Assert.assertEquals(4, partitionDirs.size());
    // The start time for the partitions should be 0, 1000, 2000, 3000
    Collections.sort(partitionDirs, Locations.LOCATION_COMPARATOR);
    for (int i = 0; i < 4; i++) {
        Assert.assertEquals(i * 1000, StreamUtils.getPartitionStartTime(partitionDirs.get(i).getName()));
    }
}