use of co.cask.cdap.data2.transaction.queue.QueueMetrics in project cdap by caskdata.
the class FlowletProgramRunner method outputEmitterFactory.
private OutputEmitterFactory outputEmitterFactory(final BasicFlowletContext flowletContext, final String flowletName, final QueueClientFactory queueClientFactory, final ImmutableList.Builder<ProducerSupplier> producerBuilder, final Table<Node, String, Set<QueueSpecification>> queueSpecs) {
return new OutputEmitterFactory() {
@Override
public <T> OutputEmitter<T> create(String outputName, TypeToken<T> type) {
try {
// first iterate over all queue specifications to find the queue name and all consumer flowlet ids
QueueName queueName = null;
List<String> consumerFlowlets = Lists.newLinkedList();
Node flowlet = Node.flowlet(flowletName);
Schema schema = schemaGenerator.generate(type.getType());
for (Map.Entry<String, Set<QueueSpecification>> entry : queueSpecs.row(flowlet).entrySet()) {
for (QueueSpecification queueSpec : entry.getValue()) {
if (queueSpec.getQueueName().getSimpleName().equals(outputName) && queueSpec.getOutputSchema().equals(schema)) {
queueName = queueSpec.getQueueName();
consumerFlowlets.add(entry.getKey());
break;
}
}
}
if (queueName == null) {
throw new IllegalArgumentException(String.format("No queue specification found for %s, %s", flowletName, type));
}
// create a metric collector for this queue, and also one for each consumer flowlet
final MetricsContext metrics = flowletContext.getProgramMetrics().childContext(Constants.Metrics.Tag.FLOWLET_QUEUE, outputName);
final MetricsContext producerMetrics = metrics.childContext(Constants.Metrics.Tag.PRODUCER, flowletContext.getFlowletId());
final Iterable<MetricsContext> consumerMetrics = Iterables.transform(consumerFlowlets, new Function<String, MetricsContext>() {
@Override
public MetricsContext apply(String consumer) {
return producerMetrics.childContext(Constants.Metrics.Tag.CONSUMER, consumer);
}
});
// create a queue metrics emitter that emit to all of the above collectors
ProducerSupplier producerSupplier = new ProducerSupplier(queueName, queueClientFactory, new QueueMetrics() {
@Override
public void emitEnqueue(int count) {
metrics.increment("process.events.out", count);
for (MetricsContext collector : consumerMetrics) {
collector.increment("queue.pending", count);
}
}
@Override
public void emitEnqueueBytes(int bytes) {
// no-op
}
});
producerBuilder.add(producerSupplier);
return new DatumOutputEmitter<>(producerSupplier, schema, datumWriterFactory.create(type, schema));
} catch (Exception e) {
throw Throwables.propagate(e);
}
}
};
}
use of co.cask.cdap.data2.transaction.queue.QueueMetrics in project cdap by caskdata.
the class HBaseQueueClientFactory method createProducer.
@Override
public QueueProducer createProducer(QueueName queueName, QueueMetrics queueMetrics) throws IOException {
HBaseQueueAdmin admin = ensureTableExists(queueName);
try {
final List<ConsumerGroupConfig> groupConfigs = Lists.newArrayList();
try (HBaseConsumerStateStore stateStore = admin.getConsumerStateStore(queueName)) {
Transactions.createTransactionExecutor(txExecutorFactory, stateStore).execute(new Subroutine() {
@Override
public void apply() throws Exception {
stateStore.getLatestConsumerGroups(groupConfigs);
}
});
}
Preconditions.checkState(!groupConfigs.isEmpty(), "Missing consumer group information for queue %s", queueName);
HTable hTable = createHTable(admin.getDataTableId(queueName, queueAdmin.getType()));
int distributorBuckets = getDistributorBuckets(hTable.getTableDescriptor());
return createProducer(hTable, queueName, queueMetrics, new ShardedHBaseQueueStrategy(hBaseTableUtil, distributorBuckets), groupConfigs);
} catch (Exception e) {
Throwables.propagateIfPossible(e);
throw new IOException(e);
}
}
Aggregations