Examples with StreamId co.cask.cdap.proto.id.StreamId used on opensource projects

Example 76 with StreamId

use of co.cask.cdap.proto.id.StreamId in project cdap by caskdata.

the class StreamAdminTest method testOwner.

@Test
public void testOwner() throws Exception {
    // crate a stream with owner
    StreamAdmin streamAdmin = getStreamAdmin();
    OwnerAdmin ownerAdmin = getOwnerAdmin();
    grantAndAssertSuccess(FOO_NAMESPACE, USER, ImmutableSet.of(Action.WRITE));
    StreamId stream = FOO_NAMESPACE.stream("stream");
    Properties properties = new Properties();
    String ownerPrincipal = "user/somehost@somekdc.net";
    properties.put(Constants.Security.PRINCIPAL, ownerPrincipal);
    streamAdmin.create(stream, properties);
    Assert.assertTrue(streamAdmin.exists(stream));
    // Check that the owner information got stored in owner store
    Assert.assertTrue(ownerAdmin.exists(stream));
    // also verify that we are able to get owner information back in properties
    Assert.assertEquals(ownerPrincipal, streamAdmin.getProperties(stream).getOwnerPrincipal());
    // updating stream owner should fail
    try {
        streamAdmin.updateConfig(stream, new StreamProperties(1L, null, null, null, "user/somekdc.net"));
        Assert.fail();
    } catch (UnauthorizedException e) {
    // expected
    }
    // trying to create same stream with different owner should fail
    properties.put(Constants.Security.PRINCIPAL, "someOtherUser/someHost@somekdc.net");
    try {
        streamAdmin.create(stream, properties);
        Assert.fail("Should have failed to add the same stream with different owner");
    } catch (UnauthorizedException e) {
    // expected
    }
    // ensure that the previous owner still exists
    Assert.assertEquals(ownerPrincipal, streamAdmin.getProperties(stream).getOwnerPrincipal());
    // drop the stream which should also delete the owner info
    streamAdmin.drop(stream);
    Assert.assertFalse(ownerAdmin.exists(stream));
}

Example 77 with StreamId

use of co.cask.cdap.proto.id.StreamId in project cdap by caskdata.

the class StreamConsumerStateTestBase method testNamespacedStore.

@Test
public void testNamespacedStore() throws Exception {
    // Store different offsets for two streams using the same StateStoreFactory to show that
    // StateStoreFactory is capable of storing distinct states for streams with same name but different namespace
    StreamAdmin streamAdmin = getStreamAdmin();
    String streamName = "testNamespacedStore";
    StreamId streamId = TEST_NAMESPACE.stream(streamName);
    StreamId otherStreamId = OTHER_NAMESPACE.stream(streamName);
    streamAdmin.create(streamId);
    streamAdmin.create(otherStreamId);
    StreamConfig config = streamAdmin.getConfig(streamId);
    StreamConfig otherConfig = streamAdmin.getConfig(otherStreamId);
    // Creates a state with 4 offsets
    StreamConsumerState state = generateState(0L, 0, config, 0L, 4);
    StreamConsumerStateStore stateStore = createStateStore(config);
    // Create another state with more offsets for stream in different namespace
    StreamConsumerState otherState = generateState(0L, 0, otherConfig, 0L, 8);
    StreamConsumerStateStore otherStateStore = createStateStore(otherConfig);
    // Save the states.
    stateStore.save(state);
    otherStateStore.save(otherState);
    // Read the state back
    StreamConsumerState readState = stateStore.get(0, 0);
    StreamConsumerState otherReadState = otherStateStore.get(0, 0);
    Assert.assertEquals(state, readState);
    Assert.assertEquals(otherState, otherReadState);
    Assert.assertNotEquals(state, otherState);
}

Example 78 with StreamId

use of co.cask.cdap.proto.id.StreamId in project cdap by caskdata.

the class StreamConsumerStateTestBase method testRemove.

@Test
public void testRemove() throws Exception {
    StreamAdmin streamAdmin = getStreamAdmin();
    String streamName = "testRemove";
    StreamId streamId = TEST_NAMESPACE.stream(streamName);
    streamAdmin.create(streamId);
    StreamConfig config = streamAdmin.getConfig(streamId);
    // Creates 4 states of 2 groups, each with 4 offsets
    Set<StreamConsumerState> states = Sets.newHashSet();
    for (int i = 0; i < 4; i++) {
        states.add(generateState(i % 2, i, config, 0L, 4));
    }
    StreamConsumerStateStore stateStore = createStateStore(config);
    stateStore.save(states);
    // Read all states back
    Set<StreamConsumerState> readStates = Sets.newHashSet();
    stateStore.getAll(readStates);
    Assert.assertEquals(states, readStates);
    // Remove groupId 0
    Set<StreamConsumerState> removeStates = Sets.newHashSet();
    for (StreamConsumerState state : readStates) {
        if (state.getGroupId() == 0) {
            removeStates.add(state);
        }
    }
    stateStore.remove(removeStates);
    // Read all states back
    readStates.clear();
    stateStore.getAll(readStates);
    Assert.assertEquals(2, readStates.size());
    for (StreamConsumerState state : readStates) {
        Assert.assertEquals(1L, state.getGroupId());
    }
}

Example 79 with StreamId

use of co.cask.cdap.proto.id.StreamId in project cdap by caskdata.

the class TimePartitionedStreamTestBase method testTimePartition.

@Test
public void testTimePartition() throws IOException {
    // Create time partition file of 1 seconds each.
    String streamName = "stream";
    Location streamLocation = getLocationFactory().create(streamName);
    streamLocation.mkdirs();
    TimePartitionedStreamFileWriter writer = new TimePartitionedStreamFileWriter(streamLocation, 1000, "file", 100, new StreamId(NamespaceId.DEFAULT.getNamespace(), streamName), impersonator);
    // Write 2 events per millis for 3 seconds, starting at 0.5 second.
    long timeBase = 500;
    for (int i = 0; i < 3; i++) {
        for (int j = 0; j < 1000; j++) {
            long offset = i * 1000 + j;
            long timestamp = timeBase + offset;
            writer.append(StreamFileTestUtils.createEvent(timestamp, "Testing " + offset + " 0"));
            writer.append(StreamFileTestUtils.createEvent(timestamp, "Testing " + offset + " 1"));
        }
    }
    writer.close();
    // There should be four partition directory (500-1000, 1000-2000, 2000-3000, 3000-3500).
    List<Location> partitionDirs = Lists.newArrayList(streamLocation.list());
    Assert.assertEquals(4, partitionDirs.size());
    // The start time for the partitions should be 0, 1000, 2000, 3000
    Collections.sort(partitionDirs, Locations.LOCATION_COMPARATOR);
    for (int i = 0; i < 4; i++) {
        Assert.assertEquals(i * 1000, StreamUtils.getPartitionStartTime(partitionDirs.get(i).getName()));
    }
}

Example 80 with StreamId

use of co.cask.cdap.proto.id.StreamId in project cdap by caskdata.

the class StreamFileSizeFetcherTest method testFetchSize.

@Test
public void testFetchSize() throws Exception {
    final String streamName = "testFetchSize";
    StreamId streamId = NamespaceId.DEFAULT.stream(streamName);
    final int nbEvents = 100;
    StreamAdmin streamAdmin = new TestStreamAdmin(namespacedLocationFactory, Long.MAX_VALUE, 1000);
    streamAdmin.create(streamId);
    StreamConfig config = streamAdmin.getConfig(streamId);
    try {
        StreamUtils.fetchStreamFilesSize(StreamUtils.createGenerationLocation(config.getLocation(), StreamUtils.getGeneration(config)));
        Assert.fail("No stream file created yet");
    } catch (IOException e) {
    // Expected
    }
    // Creates a stream file that has no event inside
    Location partitionLocation = StreamUtils.createPartitionLocation(config.getLocation(), 0, Long.MAX_VALUE);
    Location dataLocation = StreamUtils.createStreamLocation(partitionLocation, "writer", 0, StreamFileType.EVENT);
    Location idxLocation = StreamUtils.createStreamLocation(partitionLocation, "writer", 0, StreamFileType.INDEX);
    StreamDataFileWriter writer = new StreamDataFileWriter(Locations.newOutputSupplier(dataLocation), Locations.newOutputSupplier(idxLocation), 10000L);
    // Write 100 events to the stream
    for (int i = 0; i < nbEvents; i++) {
        writer.append(StreamFileTestUtils.createEvent(i, "foo"));
    }
    writer.close();
    long size = StreamUtils.fetchStreamFilesSize(StreamUtils.createGenerationLocation(config.getLocation(), StreamUtils.getGeneration(config)));
    Assert.assertTrue(size > 0);
    Assert.assertEquals(dataLocation.length(), size);
}