Examples with ContextAwareMeter org.apache.gobblin.metrics.ContextAwareMeter used on opensource projects

Example 1 with ContextAwareMeter

use of org.apache.gobblin.metrics.ContextAwareMeter in project incubator-gobblin by apache.

the class DagManager method setActive.

/**
 * When a {@link DagManager} becomes active, it loads the serialized representations of the currently running {@link Dag}s
 * from the checkpoint directory, deserializes the {@link Dag}s and adds them to a queue to be consumed by
 * the {@link DagManagerThread}s.
 * @param active a boolean to indicate if the {@link DagManager} is the leader.
 */
public synchronized void setActive(boolean active) {
    if (this.isActive == active) {
        log.info("DagManager already {}, skipping further actions.", (!active) ? "inactive" : "active");
        return;
    }
    this.isActive = active;
    try {
        if (this.isActive) {
            log.info("Activating DagManager.");
            log.info("Scheduling {} DagManager threads", numThreads);
            // Initializing state store for persisting Dags.
            this.dagStateStore = createDagStateStore(config, topologySpecMap);
            DagStateStore failedDagStateStore = createDagStateStore(ConfigUtils.getConfigOrEmpty(config, FAILED_DAG_STATESTORE_PREFIX).withFallback(config), topologySpecMap);
            Set<String> failedDagIds = Collections.synchronizedSet(failedDagStateStore.getDagIds());
            ContextAwareMeter allSuccessfulMeter = null;
            ContextAwareMeter allFailedMeter = null;
            if (instrumentationEnabled) {
                MetricContext metricContext = Instrumented.getMetricContext(ConfigUtils.configToState(ConfigFactory.empty()), getClass());
                allSuccessfulMeter = metricContext.contextAwareMeter(MetricRegistry.name(ServiceMetricNames.GOBBLIN_SERVICE_PREFIX, ServiceMetricNames.SUCCESSFUL_FLOW_METER));
                allFailedMeter = metricContext.contextAwareMeter(MetricRegistry.name(ServiceMetricNames.GOBBLIN_SERVICE_PREFIX, ServiceMetricNames.FAILED_FLOW_METER));
            }
            // On startup, the service creates DagManagerThreads that are scheduled at a fixed rate.
            this.dagManagerThreads = new DagManagerThread[numThreads];
            for (int i = 0; i < numThreads; i++) {
                DagManagerThread dagManagerThread = new DagManagerThread(jobStatusRetriever, dagStateStore, failedDagStateStore, runQueue[i], cancelQueue[i], resumeQueue[i], instrumentationEnabled, defaultQuota, perUserQuota, failedDagIds, allSuccessfulMeter, allFailedMeter, this.defaultJobStartSlaTimeMillis);
                this.dagManagerThreads[i] = dagManagerThread;
                this.scheduledExecutorPool.scheduleAtFixedRate(dagManagerThread, 0, this.pollingInterval, TimeUnit.SECONDS);
            }
            FailedDagRetentionThread failedDagRetentionThread = new FailedDagRetentionThread(failedDagStateStore, failedDagIds, failedDagRetentionTime);
            this.scheduledExecutorPool.scheduleAtFixedRate(failedDagRetentionThread, 0, retentionPollingInterval, TimeUnit.MINUTES);
            List<Dag<JobExecutionPlan>> dags = dagStateStore.getDags();
            log.info("Loading " + dags.size() + " dags from dag state store");
            for (Dag<JobExecutionPlan> dag : dags) {
                addDag(dag, false, false);
            }
        } else {
            // Mark the DagManager inactive.
            log.info("Inactivating the DagManager. Shutting down all DagManager threads");
            this.scheduledExecutorPool.shutdown();
            // The DMThread's metrics mappings follow the lifecycle of the DMThread itself and so are lost by DM deactivation-reactivation but the RootMetricContext is a (persistent) singleton.
            // To avoid IllegalArgumentException by the RMC preventing (re-)add of a metric already known, remove all metrics that a new DMThread thread would attempt to add (in DagManagerThread::initialize) whenever running post-re-enablement
            RootMetricContext.get().removeMatching(getMetricsFilterForDagManager());
            try {
                this.scheduledExecutorPool.awaitTermination(TERMINATION_TIMEOUT, TimeUnit.SECONDS);
            } catch (InterruptedException e) {
                log.error("Exception encountered when shutting down DagManager threads.", e);
            }
        }
    } catch (IOException e) {
        log.error("Exception encountered when activating the new DagManager", e);
        throw new RuntimeException(e);
    }
}