Examples with SimpleQueueSpecificationGenerator co.cask.cdap.internal.app.queue.SimpleQueueSpecificationGenerator used on opensource projects

Example 6 with SimpleQueueSpecificationGenerator

use of co.cask.cdap.internal.app.queue.SimpleQueueSpecificationGenerator in project cdap by caskdata.

the class FlowUtils method configureQueue.

/**
   * Configures all queues being used in a flow.
   *
   * @return A Multimap from flowletId to QueueName where the flowlet is a consumer of.
   */
public static Multimap<String, QueueName> configureQueue(Program program, FlowSpecification flowSpec, final StreamAdmin streamAdmin, QueueAdmin queueAdmin, TransactionExecutorFactory txExecutorFactory) {
    // Generate all queues specifications
    ApplicationId appId = new ApplicationId(program.getNamespaceId(), program.getApplicationId());
    Table<QueueSpecificationGenerator.Node, String, Set<QueueSpecification>> queueSpecs = new SimpleQueueSpecificationGenerator(appId).create(flowSpec);
    // For each queue in the flow, gather all consumer groups information
    Multimap<QueueName, ConsumerGroupConfig> queueConfigs = HashMultimap.create();
    // Loop through each flowlet and generate the map from consumer flowlet id to queue
    ImmutableSetMultimap.Builder<String, QueueName> resultBuilder = ImmutableSetMultimap.builder();
    for (Map.Entry<String, FlowletDefinition> entry : flowSpec.getFlowlets().entrySet()) {
        String flowletId = entry.getKey();
        for (QueueSpecification queueSpec : Iterables.concat(queueSpecs.column(flowletId).values())) {
            resultBuilder.put(flowletId, queueSpec.getQueueName());
        }
    }
    // For each queue, gather all consumer groups.
    for (QueueSpecification queueSpec : Iterables.concat(queueSpecs.values())) {
        QueueName queueName = queueSpec.getQueueName();
        queueConfigs.putAll(queueName, getAllConsumerGroups(program, flowSpec, queueName, queueSpecs));
    }
    try {
        // Configure each stream consumer in the Flow. Also collects all queue configurers.
        final List<ConsumerGroupConfigurer> groupConfigurers = Lists.newArrayList();
        final Map<StreamId, Map<Long, Integer>> streamConfigurers = Maps.newHashMap();
        for (Map.Entry<QueueName, Collection<ConsumerGroupConfig>> entry : queueConfigs.asMap().entrySet()) {
            LOG.info("Queue config for {} : {}", entry.getKey(), entry.getValue());
            if (entry.getKey().isStream()) {
                Map<Long, Integer> configs = Maps.newHashMap();
                for (ConsumerGroupConfig config : entry.getValue()) {
                    configs.put(config.getGroupId(), config.getGroupSize());
                }
                streamConfigurers.put(entry.getKey().toStreamId(), configs);
            } else {
                groupConfigurers.add(new ConsumerGroupConfigurer(queueAdmin.getQueueConfigurer(entry.getKey()), entry.getValue()));
            }
        }
        // Configure queue transactionally
        try {
            Transactions.createTransactionExecutor(txExecutorFactory, groupConfigurers).execute(new TransactionExecutor.Subroutine() {

                @Override
                public void apply() throws Exception {
                    for (ConsumerGroupConfigurer configurer : groupConfigurers) {
                        configurer.configure();
                    }
                    for (Map.Entry<StreamId, Map<Long, Integer>> entry : streamConfigurers.entrySet()) {
                        streamAdmin.configureGroups(entry.getKey(), entry.getValue());
                    }
                }
            });
        } finally {
            for (ConsumerGroupConfigurer configurer : groupConfigurers) {
                Closeables.closeQuietly(configurer);
            }
        }
        return resultBuilder.build();
    } catch (Exception e) {
        LOG.error("Failed to configure queues", e);
        throw Throwables.propagate(e);
    }
}

Example 7 with SimpleQueueSpecificationGenerator

use of co.cask.cdap.internal.app.queue.SimpleQueueSpecificationGenerator in project cdap by caskdata.

the class FlowQueuePendingCorrector method run.

public void run(final FlowId flowId, FlowSpecification flow) throws Exception {
    /* Temporary check. Can be removed when FlowId class is ready */
    Preconditions.checkArgument(ProgramType.FLOW == flowId.getType(), "Unexpected program type %s, FlowPendingQueueCorrector only runs on flows", flowId.getType());
    System.out.println("Running queue.pending correction on flow " + flowId);
    SimpleQueueSpecificationGenerator queueSpecGenerator = new SimpleQueueSpecificationGenerator(flowId.getParent());
    Table<QueueSpecificationGenerator.Node, String, Set<QueueSpecification>> table = queueSpecGenerator.create(flow);
    for (Table.Cell<QueueSpecificationGenerator.Node, String, Set<QueueSpecification>> cell : table.cellSet()) {
        if (cell.getRowKey().getType() == FlowletConnection.Type.FLOWLET) {
            String producerFlowlet = cell.getRowKey().getName();
            String consumerFlowlet = cell.getColumnKey();
            for (QueueSpecification queue : cell.getValue()) {
                run(flowId, producerFlowlet, consumerFlowlet, queue.getQueueName().getSimpleName());
            }
        }
    }
}