Search in sources :

Example 1 with QueueMetrics

use of co.cask.cdap.data2.transaction.queue.QueueMetrics in project cdap by caskdata.

the class FlowletProgramRunner method outputEmitterFactory.

private OutputEmitterFactory outputEmitterFactory(final BasicFlowletContext flowletContext, final String flowletName, final QueueClientFactory queueClientFactory, final ImmutableList.Builder<ProducerSupplier> producerBuilder, final Table<Node, String, Set<QueueSpecification>> queueSpecs) {
    return new OutputEmitterFactory() {

        @Override
        public <T> OutputEmitter<T> create(String outputName, TypeToken<T> type) {
            try {
                // first iterate over all queue specifications to find the queue name and all consumer flowlet ids
                QueueName queueName = null;
                List<String> consumerFlowlets = Lists.newLinkedList();
                Node flowlet = Node.flowlet(flowletName);
                Schema schema = schemaGenerator.generate(type.getType());
                for (Map.Entry<String, Set<QueueSpecification>> entry : queueSpecs.row(flowlet).entrySet()) {
                    for (QueueSpecification queueSpec : entry.getValue()) {
                        if (queueSpec.getQueueName().getSimpleName().equals(outputName) && queueSpec.getOutputSchema().equals(schema)) {
                            queueName = queueSpec.getQueueName();
                            consumerFlowlets.add(entry.getKey());
                            break;
                        }
                    }
                }
                if (queueName == null) {
                    throw new IllegalArgumentException(String.format("No queue specification found for %s, %s", flowletName, type));
                }
                // create a metric collector for this queue, and also one for each consumer flowlet
                final MetricsContext metrics = flowletContext.getProgramMetrics().childContext(Constants.Metrics.Tag.FLOWLET_QUEUE, outputName);
                final MetricsContext producerMetrics = metrics.childContext(Constants.Metrics.Tag.PRODUCER, flowletContext.getFlowletId());
                final Iterable<MetricsContext> consumerMetrics = Iterables.transform(consumerFlowlets, new Function<String, MetricsContext>() {

                    @Override
                    public MetricsContext apply(String consumer) {
                        return producerMetrics.childContext(Constants.Metrics.Tag.CONSUMER, consumer);
                    }
                });
                // create a queue metrics emitter that emit to all of the above collectors
                ProducerSupplier producerSupplier = new ProducerSupplier(queueName, queueClientFactory, new QueueMetrics() {

                    @Override
                    public void emitEnqueue(int count) {
                        metrics.increment("process.events.out", count);
                        for (MetricsContext collector : consumerMetrics) {
                            collector.increment("queue.pending", count);
                        }
                    }

                    @Override
                    public void emitEnqueueBytes(int bytes) {
                    // no-op
                    }
                });
                producerBuilder.add(producerSupplier);
                return new DatumOutputEmitter<>(producerSupplier, schema, datumWriterFactory.create(type, schema));
            } catch (Exception e) {
                throw Throwables.propagate(e);
            }
        }
    };
}
Also used : Set(java.util.Set) ImmutableSet(com.google.common.collect.ImmutableSet) Node(co.cask.cdap.app.queue.QueueSpecificationGenerator.Node) Schema(co.cask.cdap.api.data.schema.Schema) MetricsContext(co.cask.cdap.api.metrics.MetricsContext) UnsupportedTypeException(co.cask.cdap.api.data.schema.UnsupportedTypeException) IOException(java.io.IOException) QueueMetrics(co.cask.cdap.data2.transaction.queue.QueueMetrics) TypeToken(com.google.common.reflect.TypeToken) QueueSpecification(co.cask.cdap.app.queue.QueueSpecification) QueueName(co.cask.cdap.common.queue.QueueName) Map(java.util.Map)

Example 2 with QueueMetrics

use of co.cask.cdap.data2.transaction.queue.QueueMetrics in project cdap by caskdata.

the class HBaseQueueClientFactory method createProducer.

@Override
public QueueProducer createProducer(QueueName queueName, QueueMetrics queueMetrics) throws IOException {
    HBaseQueueAdmin admin = ensureTableExists(queueName);
    try {
        final List<ConsumerGroupConfig> groupConfigs = Lists.newArrayList();
        try (HBaseConsumerStateStore stateStore = admin.getConsumerStateStore(queueName)) {
            Transactions.createTransactionExecutor(txExecutorFactory, stateStore).execute(new Subroutine() {

                @Override
                public void apply() throws Exception {
                    stateStore.getLatestConsumerGroups(groupConfigs);
                }
            });
        }
        Preconditions.checkState(!groupConfigs.isEmpty(), "Missing consumer group information for queue %s", queueName);
        HTable hTable = createHTable(admin.getDataTableId(queueName, queueAdmin.getType()));
        int distributorBuckets = getDistributorBuckets(hTable.getTableDescriptor());
        return createProducer(hTable, queueName, queueMetrics, new ShardedHBaseQueueStrategy(hBaseTableUtil, distributorBuckets), groupConfigs);
    } catch (Exception e) {
        Throwables.propagateIfPossible(e);
        throw new IOException(e);
    }
}
Also used : Subroutine(org.apache.tephra.TransactionExecutor.Subroutine) IOException(java.io.IOException) HTable(org.apache.hadoop.hbase.client.HTable) ConsumerGroupConfig(co.cask.cdap.data2.queue.ConsumerGroupConfig) IOException(java.io.IOException)

Aggregations

IOException (java.io.IOException)2 Schema (co.cask.cdap.api.data.schema.Schema)1 UnsupportedTypeException (co.cask.cdap.api.data.schema.UnsupportedTypeException)1 MetricsContext (co.cask.cdap.api.metrics.MetricsContext)1 QueueSpecification (co.cask.cdap.app.queue.QueueSpecification)1 Node (co.cask.cdap.app.queue.QueueSpecificationGenerator.Node)1 QueueName (co.cask.cdap.common.queue.QueueName)1 ConsumerGroupConfig (co.cask.cdap.data2.queue.ConsumerGroupConfig)1 QueueMetrics (co.cask.cdap.data2.transaction.queue.QueueMetrics)1 ImmutableSet (com.google.common.collect.ImmutableSet)1 TypeToken (com.google.common.reflect.TypeToken)1 Map (java.util.Map)1 Set (java.util.Set)1 HTable (org.apache.hadoop.hbase.client.HTable)1 Subroutine (org.apache.tephra.TransactionExecutor.Subroutine)1