Examples with StreamId co.cask.cdap.proto.id.StreamId used on opensource projects

Example 46 with StreamId

use of co.cask.cdap.proto.id.StreamId in project cdap by caskdata.

the class SetStreamNotificationThresholdCommand method perform.

@Override
public void perform(Arguments arguments, PrintStream output) throws Exception {
    StreamId streamId = cliConfig.getCurrentNamespace().stream(arguments.get(ArgumentName.STREAM.toString()));
    int notificationThresholdMB = arguments.getInt(ArgumentName.NOTIFICATION_THRESHOLD_MB.toString());
    streamClient.setStreamProperties(streamId, new StreamProperties(null, null, notificationThresholdMB));
    output.printf("Successfully set notification threshold of stream '%s' to %dMB\n", streamId.getEntityName(), notificationThresholdMB);
}

Example 47 with StreamId

use of co.cask.cdap.proto.id.StreamId in project cdap by caskdata.

the class StreamConsumerTestBase method testNamespacedStreamConsumers.

@Test
public void testNamespacedStreamConsumers() throws Exception {
    // Test two consumers for two streams with the same name, but in different namespaces. Their consumption should be
    // independent of the other.
    String stream = "testNamespacedStreamConsumers";
    StreamId streamId = TEST_NAMESPACE.stream(stream);
    StreamId otherStreamId = OTHER_NAMESPACE.stream(stream);
    StreamAdmin streamAdmin = getStreamAdmin();
    streamAdmin.create(streamId);
    streamAdmin.create(otherStreamId);
    StreamConfig streamConfig = streamAdmin.getConfig(streamId);
    StreamConfig otherStreamConfig = streamAdmin.getConfig(otherStreamId);
    // Writes 5 events to both streams
    writeEvents(streamConfig, "Testing ", 5);
    writeEvents(otherStreamConfig, "Testing ", 5);
    streamAdmin.configureInstances(streamId, 0L, 1);
    streamAdmin.configureInstances(otherStreamId, 0L, 1);
    StreamConsumerFactory consumerFactory = getConsumerFactory();
    StreamConsumer consumer = consumerFactory.create(streamId, "fifo.rollback", new ConsumerConfig(0L, 0, 1, DequeueStrategy.FIFO, null));
    StreamConsumer otherConsumer = consumerFactory.create(otherStreamId, "fifo.rollback", new ConsumerConfig(0L, 0, 1, DequeueStrategy.FIFO, null));
    // Try to dequeue using both consumers
    TransactionContext context = createTxContext(consumer);
    TransactionContext otherContext = createTxContext(otherConsumer);
    context.start();
    otherContext.start();
    // Consume events from the stream in the default namespace
    DequeueResult<StreamEvent> result0 = consumer.poll(1, 1, TimeUnit.SECONDS);
    Assert.assertEquals("Testing 0", Charsets.UTF_8.decode(result0.iterator().next().getBody()).toString());
    context.finish();
    context.start();
    result0 = consumer.poll(1, 1, TimeUnit.SECONDS);
    Assert.assertEquals("Testing 1", Charsets.UTF_8.decode(result0.iterator().next().getBody()).toString());
    context.finish();
    context.start();
    result0 = consumer.poll(1, 1, TimeUnit.SECONDS);
    Assert.assertEquals("Testing 2", Charsets.UTF_8.decode(result0.iterator().next().getBody()).toString());
    context.finish();
    context.start();
    // Even though a stream with the same name has already consumed 3 events, the otherConsumer is for a stream in a
    // different namespace, so it will still be on the initial event.
    DequeueResult<StreamEvent> result1 = otherConsumer.poll(1, 1, TimeUnit.SECONDS);
    Assert.assertEquals("Testing 0", Charsets.UTF_8.decode(result1.iterator().next().getBody()).toString());
    otherContext.finish();
    otherContext.start();
    result0 = consumer.poll(1, 1, TimeUnit.SECONDS);
    result1 = otherConsumer.poll(1, 1, TimeUnit.SECONDS);
    Assert.assertEquals("Testing 3", Charsets.UTF_8.decode(result0.iterator().next().getBody()).toString());
    Assert.assertEquals("Testing 1", Charsets.UTF_8.decode(result1.iterator().next().getBody()).toString());
    // Commit both
    context.finish();
    otherContext.finish();
    consumer.close();
    otherConsumer.close();
}

Example 48 with StreamId

use of co.cask.cdap.proto.id.StreamId in project cdap by caskdata.

the class StreamFileJanitor method cleanAll.

/**
   * Performs file cleanup for all streams.
   */
public void cleanAll() throws Exception {
    List<NamespaceMeta> namespaces = namespaceQueryAdmin.list();
    for (final NamespaceMeta namespace : namespaces) {
        final NamespaceId namespaceId = namespace.getNamespaceId();
        final Location streamBaseLocation = impersonator.doAs(namespaceId, new Callable<Location>() {

            @Override
            public Location call() throws Exception {
                return namespacedLocationFactory.get(namespaceId).append(streamBaseDirPath);
            }
        });
        boolean exists = streamBaseLocation.exists();
        if (exists) {
            // Remove everything under the deleted directory
            Location deletedLocation = StreamUtils.getDeletedLocation(streamBaseLocation);
            if (deletedLocation.exists()) {
                Locations.deleteContent(deletedLocation);
            }
        }
        if (!exists) {
            continue;
        }
        Iterable<Location> streamLocations = StreamUtils.listAllStreams(streamBaseLocation);
        for (final Location streamLocation : streamLocations) {
            final StreamId streamId = namespaceId.stream(StreamUtils.getStreamNameFromLocation(streamLocation));
            final AtomicLong ttl = new AtomicLong(0);
            if (isStreamExists(streamId)) {
                ttl.set(streamAdmin.getConfig(streamId).getTTL());
            }
            clean(streamLocation, ttl.get(), System.currentTimeMillis());
        }
    }
}

Example 49 with StreamId

use of co.cask.cdap.proto.id.StreamId in project cdap by caskdata.

the class StreamConsumerStateTestBase method testMultiStore.

@Test
public void testMultiStore() throws Exception {
    StreamAdmin streamAdmin = getStreamAdmin();
    String streamName = "testMultiStore";
    StreamId streamId = TEST_NAMESPACE.stream(streamName);
    streamAdmin.create(streamId);
    StreamConfig config = streamAdmin.getConfig(streamId);
    // Creates 4 states of 2 groups, each with 4 offsets
    Set<StreamConsumerState> states = Sets.newHashSet();
    for (int i = 0; i < 4; i++) {
        states.add(generateState(i % 2, i, config, 0L, 4));
    }
    StreamConsumerStateStore stateStore = createStateStore(config);
    stateStore.save(states);
    // Read all states back
    Set<StreamConsumerState> readStates = Sets.newHashSet();
    stateStore.getAll(readStates);
    Assert.assertEquals(states, readStates);
}

Example 50 with StreamId

use of co.cask.cdap.proto.id.StreamId in project cdap by caskdata.

the class StreamConsumerStateTestBase method testStateExists.

@Test
public void testStateExists() throws Exception {
    StreamAdmin streamAdmin = getStreamAdmin();
    String streamName = "testStateExists";
    StreamId streamId = TEST_NAMESPACE.stream(streamName);
    streamAdmin.create(streamId);
    StreamConfig config = streamAdmin.getConfig(streamId);
    StreamConsumerStateStore stateStore = createStateStore(config);
    streamAdmin.configureInstances(TEST_NAMESPACE.stream(streamName), 0L, 1);
    // Get a consumer state that is configured
    StreamConsumerState state = stateStore.get(0L, 0);
    Assert.assertNotNull(state);
    // Try to get a consumer state that not configured yet.
    state = stateStore.get(0L, 1);
    Assert.assertNull(state);
}