Examples with QueueName co.cask.cdap.common.queue.QueueName used on opensource projects

Example 21 with QueueName

use of co.cask.cdap.common.queue.QueueName in project cdap by caskdata.

the class FlowUtils method configureQueue.

/**
   * Configures all queues being used in a flow.
   *
   * @return A Multimap from flowletId to QueueName where the flowlet is a consumer of.
   */
public static Multimap<String, QueueName> configureQueue(Program program, FlowSpecification flowSpec, final StreamAdmin streamAdmin, QueueAdmin queueAdmin, TransactionExecutorFactory txExecutorFactory) {
    // Generate all queues specifications
    ApplicationId appId = new ApplicationId(program.getNamespaceId(), program.getApplicationId());
    Table<QueueSpecificationGenerator.Node, String, Set<QueueSpecification>> queueSpecs = new SimpleQueueSpecificationGenerator(appId).create(flowSpec);
    // For each queue in the flow, gather all consumer groups information
    Multimap<QueueName, ConsumerGroupConfig> queueConfigs = HashMultimap.create();
    // Loop through each flowlet and generate the map from consumer flowlet id to queue
    ImmutableSetMultimap.Builder<String, QueueName> resultBuilder = ImmutableSetMultimap.builder();
    for (Map.Entry<String, FlowletDefinition> entry : flowSpec.getFlowlets().entrySet()) {
        String flowletId = entry.getKey();
        for (QueueSpecification queueSpec : Iterables.concat(queueSpecs.column(flowletId).values())) {
            resultBuilder.put(flowletId, queueSpec.getQueueName());
        }
    }
    // For each queue, gather all consumer groups.
    for (QueueSpecification queueSpec : Iterables.concat(queueSpecs.values())) {
        QueueName queueName = queueSpec.getQueueName();
        queueConfigs.putAll(queueName, getAllConsumerGroups(program, flowSpec, queueName, queueSpecs));
    }
    try {
        // Configure each stream consumer in the Flow. Also collects all queue configurers.
        final List<ConsumerGroupConfigurer> groupConfigurers = Lists.newArrayList();
        final Map<StreamId, Map<Long, Integer>> streamConfigurers = Maps.newHashMap();
        for (Map.Entry<QueueName, Collection<ConsumerGroupConfig>> entry : queueConfigs.asMap().entrySet()) {
            LOG.info("Queue config for {} : {}", entry.getKey(), entry.getValue());
            if (entry.getKey().isStream()) {
                Map<Long, Integer> configs = Maps.newHashMap();
                for (ConsumerGroupConfig config : entry.getValue()) {
                    configs.put(config.getGroupId(), config.getGroupSize());
                }
                streamConfigurers.put(entry.getKey().toStreamId(), configs);
            } else {
                groupConfigurers.add(new ConsumerGroupConfigurer(queueAdmin.getQueueConfigurer(entry.getKey()), entry.getValue()));
            }
        }
        // Configure queue transactionally
        try {
            Transactions.createTransactionExecutor(txExecutorFactory, groupConfigurers).execute(new TransactionExecutor.Subroutine() {

                @Override
                public void apply() throws Exception {
                    for (ConsumerGroupConfigurer configurer : groupConfigurers) {
                        configurer.configure();
                    }
                    for (Map.Entry<StreamId, Map<Long, Integer>> entry : streamConfigurers.entrySet()) {
                        streamAdmin.configureGroups(entry.getKey(), entry.getValue());
                    }
                }
            });
        } finally {
            for (ConsumerGroupConfigurer configurer : groupConfigurers) {
                Closeables.closeQuietly(configurer);
            }
        }
        return resultBuilder.build();
    } catch (Exception e) {
        LOG.error("Failed to configure queues", e);
        throw Throwables.propagate(e);
    }
}

Example 22 with QueueName

use of co.cask.cdap.common.queue.QueueName in project cdap by caskdata.

the class InMemoryStreamConsumerFactory method create.

@Override
public StreamConsumer create(StreamId streamId, String namespace, ConsumerConfig consumerConfig) throws IOException {
    QueueName queueName = QueueName.fromStream(streamId);
    QueueConsumer consumer = queueClientFactory.createConsumer(queueName, consumerConfig, -1);
    return new QueueToStreamConsumer(streamId, consumerConfig, consumer);
}

Example 23 with QueueName

use of co.cask.cdap.common.queue.QueueName in project cdap by caskdata.

the class QueueTest method testBatchHash.

// Batch enqueue and batch dequeue with one consumer.
@Category(SlowTests.class)
@Test(timeout = TIMEOUT_MS)
public void testBatchHash() throws Exception {
    QueueName queueName = QueueName.fromFlowlet(NamespaceId.DEFAULT.getEntityName(), "app", "flow", "flowlet", "batchhash");
    enqueueDequeue(queueName, 2 * ROUNDS, ROUNDS, 10, 1, DequeueStrategy.HASH, 10);
}

Example 24 with QueueName

use of co.cask.cdap.common.queue.QueueName in project cdap by caskdata.

the class QueueTest method testSingleFifo.

// Simple enqueue and dequeue with one consumer, no batch
@Test(timeout = TIMEOUT_MS)
public void testSingleFifo() throws Exception {
    QueueName queueName = QueueName.fromFlowlet(NamespaceId.DEFAULT.getEntityName(), "app", "flow", "flowlet", "singlefifo");
    enqueueDequeue(queueName, ROUNDS, ROUNDS, 1, 1, DequeueStrategy.FIFO, 1);
}

Example 25 with QueueName

use of co.cask.cdap.common.queue.QueueName in project cdap by caskdata.

the class QueueTest method testMultiStageConsumer.

@Test
public void testMultiStageConsumer() throws Exception {
    final QueueName queueName = QueueName.fromFlowlet(NamespaceId.DEFAULT.getEntityName(), "app", "flow", "flowlet", "multistage");
    ConsumerGroupConfig groupConfig = new ConsumerGroupConfig(0L, 2, DequeueStrategy.HASH, "key");
    configureGroups(queueName, ImmutableList.of(groupConfig));
    List<ConsumerConfig> consumerConfigs = ImmutableList.of(new ConsumerConfig(groupConfig, 0), new ConsumerConfig(groupConfig, 1));
    // Enqueue 10 items
    try (QueueProducer producer = queueClientFactory.createProducer(queueName)) {
        for (int i = 0; i < 10; i++) {
            TransactionContext txContext = createTxContext(producer);
            txContext.start();
            producer.enqueue(new QueueEntry("key", i, Bytes.toBytes(i)));
            txContext.finish();
        }
    }
    // Consumer all even entries
    try (QueueConsumer consumer = queueClientFactory.createConsumer(queueName, consumerConfigs.get(0), 1)) {
        for (int i = 0; i < 5; i++) {
            TransactionContext txContext = createTxContext(consumer);
            txContext.start();
            DequeueResult<byte[]> result = consumer.dequeue();
            Assert.assertTrue(!result.isEmpty());
            Assert.assertEquals(i * 2, Bytes.toInt(result.iterator().next()));
            txContext.finish();
        }
    }
    // Consume 2 odd entries
    try (QueueConsumer consumer = queueClientFactory.createConsumer(queueName, consumerConfigs.get(1), 1)) {
        TransactionContext txContext = createTxContext(consumer);
        txContext.start();
        DequeueResult<byte[]> result = consumer.dequeue(2);
        Assert.assertEquals(2, result.size());
        Iterator<byte[]> iter = result.iterator();
        for (int i = 0; i < 2; i++) {
            Assert.assertEquals(i * 2 + 1, Bytes.toInt(iter.next()));
        }
        txContext.finish();
    }
    // Consume the rest odd entries
    try (QueueConsumer consumer = queueClientFactory.createConsumer(queueName, consumerConfigs.get(1), 1)) {
        for (int i = 2; i < 5; i++) {
            TransactionContext txContext = createTxContext(consumer);
            txContext.start();
            DequeueResult<byte[]> result = consumer.dequeue();
            Assert.assertTrue(!result.isEmpty());
            Assert.assertEquals(i * 2 + 1, Bytes.toInt(result.iterator().next()));
            txContext.finish();
        }
    }
}