Examples with ConsumerGroupConfig co.cask.cdap.data2.queue.ConsumerGroupConfig used on opensource projects

Example 21 with ConsumerGroupConfig

use of co.cask.cdap.data2.queue.ConsumerGroupConfig in project cdap by caskdata.

the class HBaseQueueTest method testQueueUpgrade.

// This test upgrade from old queue (salted base) to new queue (sharded base)
@Test(timeout = 30000L)
public void testQueueUpgrade() throws Exception {
    final QueueName queueName = QueueName.fromFlowlet(NamespaceId.DEFAULT.getEntityName(), "app", "flow", "flowlet", "upgrade");
    HBaseQueueAdmin hbaseQueueAdmin = (HBaseQueueAdmin) queueAdmin;
    HBaseQueueClientFactory hBaseQueueClientFactory = (HBaseQueueClientFactory) queueClientFactory;
    // Create the old queue table explicitly
    HBaseQueueAdmin oldQueueAdmin = new HBaseQueueAdmin(hConf, cConf, injector.getInstance(LocationFactory.class), injector.getInstance(HBaseTableUtil.class), injector.getInstance(DatasetFramework.class), injector.getInstance(TransactionExecutorFactory.class), QueueConstants.QueueType.QUEUE, injector.getInstance(NamespaceQueryAdmin.class), injector.getInstance(Impersonator.class));
    oldQueueAdmin.create(queueName);
    int buckets = cConf.getInt(QueueConstants.ConfigKeys.QUEUE_TABLE_PRESPLITS);
    try (final HBaseQueueProducer oldProducer = hBaseQueueClientFactory.createProducer(oldQueueAdmin, queueName, QueueConstants.QueueType.QUEUE, QueueMetrics.NOOP_QUEUE_METRICS, new SaltedHBaseQueueStrategy(tableUtil, buckets), new ArrayList<ConsumerGroupConfig>())) {
        // Enqueue 10 items to old queue table
        Transactions.createTransactionExecutor(executorFactory, oldProducer).execute(new TransactionExecutor.Subroutine() {

            @Override
            public void apply() throws Exception {
                for (int i = 0; i < 10; i++) {
                    oldProducer.enqueue(new QueueEntry("key", i, Bytes.toBytes("Message " + i)));
                }
            }
        });
    }
    // Configure the consumer
    final ConsumerConfig consumerConfig = new ConsumerConfig(0L, 0, 1, DequeueStrategy.HASH, "key");
    try (QueueConfigurer configurer = queueAdmin.getQueueConfigurer(queueName)) {
        Transactions.createTransactionExecutor(executorFactory, configurer).execute(new TransactionExecutor.Subroutine() {

            @Override
            public void apply() throws Exception {
                configurer.configureGroups(ImmutableList.of(consumerConfig));
            }
        });
    }
    // explicit set the consumer state to be the lowest start row
    try (HBaseConsumerStateStore stateStore = hbaseQueueAdmin.getConsumerStateStore(queueName)) {
        Transactions.createTransactionExecutor(executorFactory, stateStore).execute(new TransactionExecutor.Subroutine() {

            @Override
            public void apply() throws Exception {
                stateStore.updateState(consumerConfig.getGroupId(), consumerConfig.getInstanceId(), QueueEntryRow.getQueueEntryRowKey(queueName, 0L, 0));
            }
        });
    }
    // Enqueue 10 more items to new queue table
    createEnqueueRunnable(queueName, 10, 1, null).run();
    // Verify both old and new table have 10 rows each
    Assert.assertEquals(10, countRows(hbaseQueueAdmin.getDataTableId(queueName, QueueConstants.QueueType.QUEUE)));
    Assert.assertEquals(10, countRows(hbaseQueueAdmin.getDataTableId(queueName, QueueConstants.QueueType.SHARDED_QUEUE)));
    // Create a consumer. It should see all 20 items
    final List<String> messages = Lists.newArrayList();
    try (final QueueConsumer consumer = queueClientFactory.createConsumer(queueName, consumerConfig, 1)) {
        while (messages.size() != 20) {
            Transactions.createTransactionExecutor(executorFactory, (TransactionAware) consumer).execute(new TransactionExecutor.Subroutine() {

                @Override
                public void apply() throws Exception {
                    DequeueResult<byte[]> result = consumer.dequeue(20);
                    for (byte[] data : result) {
                        messages.add(Bytes.toString(data));
                    }
                }
            });
        }
    }
    verifyQueueIsEmpty(queueName, ImmutableList.of(consumerConfig));
}

Example 22 with ConsumerGroupConfig

use of co.cask.cdap.data2.queue.ConsumerGroupConfig in project cdap by caskdata.

the class FlowUtils method getAllConsumerGroups.

/**
 * Gets all consumer group configurations for the given queue.
 */
private static Set<ConsumerGroupConfig> getAllConsumerGroups(Program program, FlowSpecification flowSpec, QueueName queueName, Table<QueueSpecificationGenerator.Node, String, Set<QueueSpecification>> queueSpecs) {
    Set<ConsumerGroupConfig> groupConfigs = Sets.newHashSet();
    SchemaGenerator schemaGenerator = new ReflectionSchemaGenerator();
    // Get all the consumers of this queue.
    for (Map.Entry<String, FlowletDefinition> entry : flowSpec.getFlowlets().entrySet()) {
        String flowletId = entry.getKey();
        for (QueueSpecification queueSpec : Iterables.concat(queueSpecs.column(flowletId).values())) {
            if (!queueSpec.getQueueName().equals(queueName)) {
                continue;
            }
            try {
                // Inspect the flowlet consumer
                FlowletDefinition flowletDefinition = entry.getValue();
                Class<?> flowletClass = program.getClassLoader().loadClass(flowletDefinition.getFlowletSpec().getClassName());
                long groupId = generateConsumerGroupId(program.getId(), flowletId);
                addConsumerGroup(queueSpec, flowletClass, groupId, flowletDefinition.getInstances(), schemaGenerator, groupConfigs);
            } catch (ClassNotFoundException e) {
                // There is no way for not able to load a Flowlet class as it should be verified during deployment.
                throw Throwables.propagate(e);
            }
        }
    }
    return groupConfigs;
}

Example 23 with ConsumerGroupConfig

use of co.cask.cdap.data2.queue.ConsumerGroupConfig in project cdap by caskdata.

the class FlowUtils method configureQueue.

/**
 * Configures all queues being used in a flow.
 *
 * @return A Multimap from flowletId to QueueName where the flowlet is a consumer of.
 */
public static Multimap<String, QueueName> configureQueue(Program program, FlowSpecification flowSpec, final StreamAdmin streamAdmin, QueueAdmin queueAdmin, TransactionExecutorFactory txExecutorFactory) {
    // Generate all queues specifications
    ApplicationId appId = new ApplicationId(program.getNamespaceId(), program.getApplicationId());
    Table<QueueSpecificationGenerator.Node, String, Set<QueueSpecification>> queueSpecs = new SimpleQueueSpecificationGenerator(appId).create(flowSpec);
    // For each queue in the flow, gather all consumer groups information
    Multimap<QueueName, ConsumerGroupConfig> queueConfigs = HashMultimap.create();
    // Loop through each flowlet and generate the map from consumer flowlet id to queue
    ImmutableSetMultimap.Builder<String, QueueName> resultBuilder = ImmutableSetMultimap.builder();
    for (Map.Entry<String, FlowletDefinition> entry : flowSpec.getFlowlets().entrySet()) {
        String flowletId = entry.getKey();
        for (QueueSpecification queueSpec : Iterables.concat(queueSpecs.column(flowletId).values())) {
            resultBuilder.put(flowletId, queueSpec.getQueueName());
        }
    }
    // For each queue, gather all consumer groups.
    for (QueueSpecification queueSpec : Iterables.concat(queueSpecs.values())) {
        QueueName queueName = queueSpec.getQueueName();
        queueConfigs.putAll(queueName, getAllConsumerGroups(program, flowSpec, queueName, queueSpecs));
    }
    try {
        // Configure each stream consumer in the Flow. Also collects all queue configurers.
        final List<ConsumerGroupConfigurer> groupConfigurers = Lists.newArrayList();
        final Map<StreamId, Map<Long, Integer>> streamConfigurers = Maps.newHashMap();
        for (Map.Entry<QueueName, Collection<ConsumerGroupConfig>> entry : queueConfigs.asMap().entrySet()) {
            LOG.info("Queue config for {} : {}", entry.getKey(), entry.getValue());
            if (entry.getKey().isStream()) {
                Map<Long, Integer> configs = Maps.newHashMap();
                for (ConsumerGroupConfig config : entry.getValue()) {
                    configs.put(config.getGroupId(), config.getGroupSize());
                }
                streamConfigurers.put(entry.getKey().toStreamId(), configs);
            } else {
                groupConfigurers.add(new ConsumerGroupConfigurer(queueAdmin.getQueueConfigurer(entry.getKey()), entry.getValue()));
            }
        }
        // Configure queue transactionally
        try {
            Transactions.createTransactionExecutor(txExecutorFactory, groupConfigurers).execute(new TransactionExecutor.Subroutine() {

                @Override
                public void apply() throws Exception {
                    for (ConsumerGroupConfigurer configurer : groupConfigurers) {
                        configurer.configure();
                    }
                    for (Map.Entry<StreamId, Map<Long, Integer>> entry : streamConfigurers.entrySet()) {
                        streamAdmin.configureGroups(entry.getKey(), entry.getValue());
                    }
                }
            });
        } finally {
            for (ConsumerGroupConfigurer configurer : groupConfigurers) {
                Closeables.closeQuietly(configurer);
            }
        }
        return resultBuilder.build();
    } catch (Exception e) {
        LOG.error("Failed to configure queues", e);
        throw Throwables.propagate(e);
    }
}