Examples with QueueName co.cask.cdap.common.queue.QueueName used on opensource projects

Example 16 with QueueName

use of co.cask.cdap.common.queue.QueueName in project cdap by caskdata.

the class DistributedFlowProgramRunner method getFlowletQueues.

/**
   * Gets the queue configuration of the Flow based on the connections in the given {@link FlowSpecification}.
   */
private Multimap<String, QueueName> getFlowletQueues(ApplicationId appId, FlowSpecification flowSpec) {
    // Generate all queues specifications
    Table<QueueSpecificationGenerator.Node, String, Set<QueueSpecification>> queueSpecs = new SimpleQueueSpecificationGenerator(appId).create(flowSpec);
    // For storing result from flowletId to queue.
    ImmutableSetMultimap.Builder<String, QueueName> resultBuilder = ImmutableSetMultimap.builder();
    // Loop through each flowlet
    for (Map.Entry<String, FlowletDefinition> entry : flowSpec.getFlowlets().entrySet()) {
        String flowletId = entry.getKey();
        // For each queue that the flowlet is a consumer, store the number of instances for this flowlet
        for (QueueSpecification queueSpec : Iterables.concat(queueSpecs.column(flowletId).values())) {
            resultBuilder.put(flowletId, queueSpec.getQueueName());
        }
    }
    return resultBuilder.build();
}

Example 17 with QueueName

use of co.cask.cdap.common.queue.QueueName in project cdap by caskdata.

the class HBaseQueueDebugger method main.

public static void main(String[] args) throws Exception {
    if (args.length >= 1 && args[0].equals("help")) {
        System.out.println("Arguments: [<queue-uri> [consumer-flowlet]]");
        System.out.println("queue-uri: queue:///<namespace>/<app>/<flow>/<flowlet>/<queue>");
        System.out.println("consumer-flowlet: <flowlet>");
        System.out.println("If queue-uri is not provided, scan all queues");
        System.out.println("Example: queue:///default/PurchaseHistory/PurchaseFlow/reader/queue collector");
        System.out.println();
        System.out.println("System properties:");
        System.out.println("-D" + PROP_SHOW_PROGRESS + "=true         Show progress while scanning the queue table");
        System.out.println("-D" + PROP_ROWS_CACHE + "=[num_of_rows]   " + "Number of rows to pass to HBase Scan.setCaching() method");
        System.exit(1);
    }
    // e.g. "queue:///default/PurchaseHistory/PurchaseFlow/reader/queue"
    final QueueName queueName = args.length >= 1 ? QueueName.from(URI.create(args[0])) : null;
    Long consumerGroupId = null;
    if (args.length >= 2) {
        Preconditions.checkNotNull(queueName);
        String consumerFlowlet = args[1];
        FlowId flowId = new FlowId(queueName.getFirstComponent(), queueName.getSecondComponent(), queueName.getThirdComponent());
        consumerGroupId = FlowUtils.generateConsumerGroupId(flowId, consumerFlowlet);
    }
    final HBaseQueueDebugger debugger = createDebugger();
    debugger.startAndWait();
    // CDAP-9005 We need to create the NamespaceQueryAdmin without authorization enabled, but create the
    // HBaseQueueDebugger with authorization enabled.
    Injector injector = createInjector(true);
    NoAuthService noAuthService = injector.getInstance(NoAuthService.class);
    noAuthService.startAndWait();
    NamespaceQueryAdmin namespaceQueryAdmin = noAuthService.getNamespaceQueryAdmin();
    Impersonator impersonator = noAuthService.getImpersonator();
    if (queueName != null) {
        final Long finalConsumerGroupId = consumerGroupId;
        impersonator.doAs(new NamespaceId(queueName.getFirstComponent()), new Callable<Void>() {

            @Override
            public Void call() throws Exception {
                debugger.scanQueue(queueName, finalConsumerGroupId);
                return null;
            }
        });
    } else {
        debugger.scanQueues(namespaceQueryAdmin.list());
    }
    noAuthService.stopAndWait();
    debugger.stopAndWait();
}

Example 18 with QueueName

use of co.cask.cdap.common.queue.QueueName in project cdap by caskdata.

the class FlowQueuePendingCorrector method run.

/**
   * Corrects queue.pending metric for a flowlet.
   */
public void run(FlowId flowId, String producerFlowlet, String consumerFlowlet, String flowletQueue, FlowSpecification flow) throws Exception {
    System.out.println("Running queue.pending correction on flow '" + flowId + "' producerFlowlet '" + producerFlowlet + "' consumerFlowlet '" + consumerFlowlet + "' flowletQueue '" + flowletQueue + "'");
    Map<RunId, ProgramRuntimeService.RuntimeInfo> runtimeInfos = programRuntimeService.list(flowId);
    Preconditions.checkState(runtimeInfos.isEmpty(), "Cannot run tool when flow " + flowId + " is still running");
    SimpleQueueSpecificationGenerator queueSpecGenerator = new SimpleQueueSpecificationGenerator(flowId.getParent());
    Table<QueueSpecificationGenerator.Node, String, Set<QueueSpecification>> table = queueSpecGenerator.create(flow);
    Preconditions.checkArgument(table.contains(QueueSpecificationGenerator.Node.flowlet(producerFlowlet), consumerFlowlet), "Flowlet " + producerFlowlet + " is not emitting to " + consumerFlowlet);
    Set<QueueSpecification> queueSpecs = table.get(QueueSpecificationGenerator.Node.flowlet(producerFlowlet), consumerFlowlet);
    boolean validQueue = false;
    for (QueueSpecification queueSpec : queueSpecs) {
        if (queueSpec.getQueueName().getSimpleName().equals(flowletQueue)) {
            validQueue = true;
            break;
        }
    }
    Preconditions.checkArgument(validQueue, "Queue " + flowletQueue + " does not exist for the given flowlets");
    QueueName queueName = QueueName.fromFlowlet(flowId, producerFlowlet, flowletQueue);
    long consumerGroupId = FlowUtils.generateConsumerGroupId(flowId, consumerFlowlet);
    long correctQueuePendingValue;
    try {
        HBaseQueueDebugger.QueueStatistics stats = queueDebugger.scanQueue(queueName, consumerGroupId);
        correctQueuePendingValue = stats.getUnprocessed() + stats.getProcessedAndNotVisible();
    } catch (NotFoundException e) {
        // OK since flowlet queue exists, but actual queue doesn't exist
        // (e.g. when running upgrade tool from 2.8 to 3.0)
        correctQueuePendingValue = 0;
    }
    Map<String, String> tags = ImmutableMap.<String, String>builder().put(Constants.Metrics.Tag.NAMESPACE, flowId.getNamespace()).put(Constants.Metrics.Tag.APP, flowId.getApplication()).put(Constants.Metrics.Tag.FLOW, flowId.getProgram()).put(Constants.Metrics.Tag.CONSUMER, consumerFlowlet).put(Constants.Metrics.Tag.PRODUCER, producerFlowlet).put(Constants.Metrics.Tag.FLOWLET_QUEUE, flowletQueue).build();
    MetricDataQuery query = new MetricDataQuery(0, 0, Integer.MAX_VALUE, 1, ImmutableMap.of("system.queue.pending", AggregationFunction.SUM), tags, ImmutableList.<String>of(), null);
    Collection<MetricTimeSeries> results = metricStore.query(query);
    long queuePending;
    if (results.isEmpty()) {
        queuePending = 0;
    } else {
        System.out.println("Got results: " + GSON.toJson(results));
        Preconditions.checkState(results.size() == 1);
        List<TimeValue> timeValues = results.iterator().next().getTimeValues();
        Preconditions.checkState(timeValues.size() == 1);
        TimeValue timeValue = timeValues.get(0);
        queuePending = timeValue.getValue();
    }
    metricsCollectionService.startAndWait();
    MetricsContext collector = metricsCollectionService.getContext(tags);
    collector.gauge("queue.pending", correctQueuePendingValue);
    System.out.printf("Adjusted system.queue.pending metric from %d to %d (tags %s)\n", queuePending, correctQueuePendingValue, GSON.toJson(tags));
    // stop will flush the metrics
    metricsCollectionService.stopAndWait();
}

Example 19 with QueueName

use of co.cask.cdap.common.queue.QueueName in project cdap by caskdata.

the class MetricStoreRequestExecutor method computeFlowletPending.

private Object computeFlowletPending(MetricDataQuery query) throws Exception {
    // Pending is processed by flowlet minus emitted into the queues it was processing from.
    // trick: get all queues and streams it was processing from using group by
    MetricDataQuery groupByQueueName = new MetricDataQuery(new MetricDataQuery(query, "system.process.events.processed", AggregationFunction.SUM), ImmutableList.of(Constants.Metrics.Tag.FLOWLET_QUEUE));
    Map<String, Long> processedPerQueue = getTotalsWithSingleGroupByTag(groupByQueueName);
    long processedTotal = 0;
    long writtenTotal = 0;
    for (Map.Entry<String, Long> entry : processedPerQueue.entrySet()) {
        String name = entry.getKey();
        // note: each has "input." prefix
        QueueName queueName = QueueName.from(URI.create(name.substring("input.".length(), name.length())));
        long written;
        if (queueName.isQueue()) {
            Map<String, String> sliceByTags = Maps.newHashMap(query.getSliceByTags());
            // we want to aggregate written to the queue by all flowlets
            sliceByTags.remove(Constants.Metrics.Tag.FLOWLET);
            // we want to narrow down to specific queue we know our flowlet was consuming from
            sliceByTags.put(Constants.Metrics.Tag.FLOWLET_QUEUE, queueName.getSimpleName());
            written = getTotals(new MetricDataQuery(new MetricDataQuery(query, sliceByTags), "system.process.events.out", AggregationFunction.SUM));
        } else if (queueName.isStream()) {
            Map<String, String> sliceByTags = Maps.newHashMap();
            // we want to narrow down to specific stream we know our flowlet was consuming from
            sliceByTags.put(Constants.Metrics.Tag.STREAM, queueName.getSimpleName());
            // note: namespace + stream uniquely define the stream
            // we know that flow can consume from stream of the same namespace only at this point
            sliceByTags.put(Constants.Metrics.Tag.NAMESPACE, query.getSliceByTags().get(Constants.Metrics.Tag.NAMESPACE));
            written = getTotals(new MetricDataQuery(new MetricDataQuery(query, sliceByTags), "system.collect.events", AggregationFunction.SUM));
        } else {
            LOG.warn("Unknown queue type: " + name);
            continue;
        }
        processedTotal += entry.getValue();
        writtenTotal += written;
    }
    long pending = writtenTotal - processedTotal;
    return new AggregateResponse(pending > 0 ? pending : 0);
}

Example 20 with QueueName

use of co.cask.cdap.common.queue.QueueName in project cdap by caskdata.

the class FlowletProcessDriver method processMethodCallback.

private <T> ProcessMethodCallback processMethodCallback(final PriorityQueue<FlowletProcessEntry<?>> processQueue, final FlowletProcessEntry<T> processEntry, final InputDatum<T> input) {
    // If it is generator flowlet, processCount is 1.
    final int processedCount = processEntry.getProcessSpec().getProcessMethod().needsInput() ? input.size() : 1;
    return new ProcessMethodCallback() {

        private final LoadingCache<String, MetricsContext> queueMetricsCollectors = CacheBuilder.newBuilder().expireAfterAccess(1, TimeUnit.HOURS).build(new CacheLoader<String, MetricsContext>() {

            @Override
            public MetricsContext load(String key) throws Exception {
                return flowletContext.getProgramMetrics().childContext(Constants.Metrics.Tag.FLOWLET_QUEUE, key);
            }
        });

        @Override
        public void onSuccess(Object object, InputContext inputContext) {
            try {
                gaugeEventProcessed(input.getQueueName());
                txCallback.onSuccess(object, inputContext);
            } catch (Throwable t) {
                LOG.error("Exception on onSuccess call: {}", flowletContext, t);
            } finally {
                enqueueEntry();
            }
        }

        @Override
        public void onFailure(Object inputObject, InputContext inputContext, FailureReason reason, InputAcknowledger inputAcknowledger) {
            LOG.warn("Process failure: {}, {}, input: {}", flowletContext, reason.getMessage(), input, reason.getCause());
            FailurePolicy failurePolicy;
            try {
                flowletContext.getProgramMetrics().increment("process.errors", 1);
                failurePolicy = txCallback.onFailure(inputObject, inputContext, reason);
                if (failurePolicy == null) {
                    failurePolicy = FailurePolicy.RETRY;
                    LOG.info("Callback returns null for failure policy. Default to {}.", failurePolicy);
                }
            } catch (Throwable t) {
                LOG.error("Exception on onFailure call: {}", flowletContext, t);
                failurePolicy = FailurePolicy.RETRY;
            }
            if (input.getRetry() >= processEntry.getProcessSpec().getProcessMethod().getMaxRetries()) {
                LOG.info("Too many retries, ignores the input: {}", input);
                failurePolicy = FailurePolicy.IGNORE;
            }
            if (failurePolicy == FailurePolicy.RETRY) {
                FlowletProcessEntry retryEntry = processEntry.isRetry() ? processEntry : FlowletProcessEntry.create(processEntry.getProcessSpec(), new ProcessSpecification<>(new SingleItemQueueReader<>(input), processEntry.getProcessSpec().getProcessMethod(), null));
                processQueue.offer(retryEntry);
            } else if (failurePolicy == FailurePolicy.IGNORE) {
                try {
                    gaugeEventProcessed(input.getQueueName());
                    inputAcknowledger.ack();
                } catch (Throwable t) {
                    LOG.error("Fatal problem, fail to ack an input: {}", flowletContext, t);
                } finally {
                    enqueueEntry();
                }
            }
        }

        private void enqueueEntry() {
            processQueue.offer(processEntry.resetRetry());
        }

        private void gaugeEventProcessed(QueueName inputQueueName) {
            if (processEntry.isTick()) {
                flowletContext.getProgramMetrics().increment("process.ticks.processed", processedCount);
            } else if (inputQueueName == null) {
                flowletContext.getProgramMetrics().increment("process.events.processed", processedCount);
            } else {
                queueMetricsCollectors.getUnchecked(inputQueueName.getSimpleName()).increment("process.events.processed", processedCount);
            }
        }
    };
}