Examples with NotConfiguredException org.apache.gobblin.broker.iface.NotConfiguredException used on opensource projects

Example 1 with NotConfiguredException

use of org.apache.gobblin.broker.iface.NotConfiguredException in project incubator-gobblin by apache.

the class MRStressTest method createLimiter.

static Limiter createLimiter(Configuration configuration, SharedResourcesBroker<SimpleScopeType> broker) {
    try {
        Limiter limiter = new NoopLimiter();
        long localQps = configuration.getLong(LOCALLY_ENFORCED_QPS, 0);
        if (localQps > 0) {
            log.info("Setting up local qps " + localQps);
            limiter = new MultiLimiter(limiter, new RateBasedLimiter(localQps));
        }
        if (configuration.getBoolean(USE_THROTTLING_SERVER, false)) {
            log.info("Setting up remote throttling.");
            String resourceId = configuration.get(RESOURCE_ID);
            Limiter globalLimiter = broker.getSharedResource(new RestliLimiterFactory<SimpleScopeType>(), new SharedLimiterKey(resourceId));
            limiter = new MultiLimiter(limiter, globalLimiter);
        }
        return limiter;
    } catch (NotConfiguredException nce) {
        throw new RuntimeException(nce);
    }
}

Example 2 with NotConfiguredException

use of org.apache.gobblin.broker.iface.NotConfiguredException in project incubator-gobblin by apache.

the class LimiterServerResource method get.

/**
 * Request permits from the limiter server. The returned {@link PermitAllocation} specifies the number of permits
 * that the client can use.
 */
@Override
@RestMethod.Get
public void get(ComplexResourceKey<PermitRequest, EmptyRecord> key, @CallbackParam final Callback<PermitAllocation> callback) {
    try (Closeable context = this.requestTimer == null ? NoopCloseable.INSTANCE : this.requestTimer.time()) {
        PermitRequest request = key.getKey();
        String resourceId = request.getResource();
        MetricContext resourceContext = (MetricContext) broker.getSharedResource(new MetricContextFactory(), new SubTaggedMetricContextKey(resourceId, ImmutableMap.of(RESOURCE_ID_TAG, resourceId)));
        Meter permitsRequestedMeter = resourceContext.meter(PERMITS_REQUESTED_METER_NAME);
        Meter permitsGrantedMeter = resourceContext.meter(PERMITS_GRANTED_METER_NAME);
        Timer limiterTimer = resourceContext.timer(LIMITER_TIMER_NAME);
        permitsRequestedMeter.mark(request.getPermits());
        if (this.leaderFinderOpt.isPresent() && !this.leaderFinderOpt.get().isLeader()) {
            URI leaderUri = this.leaderFinderOpt.get().getLeaderMetadata().getUri();
            RestLiServiceException exception = new RestLiServiceException(HttpStatus.S_301_MOVED_PERMANENTLY, String.format("New leader <a href=\"%s\">%s</a>", leaderUri, leaderUri));
            exception.setErrorDetails(new DataMap(ImmutableMap.of(LOCATION_301, leaderUri.toString())));
            throw exception;
        } else {
            ThrottlingPolicy policy = (ThrottlingPolicy) this.broker.getSharedResource(new ThrottlingPolicyFactory(), new SharedLimiterKey(request.getResource()));
            PermitAllocation allocation;
            try (Closeable thisContext = limiterTimer.time()) {
                allocation = policy.computePermitAllocation(request);
            }
            permitsGrantedMeter.mark(allocation.getPermits());
            callback.onSuccess(allocation);
        }
    } catch (NotConfiguredException nce) {
        throw new RestLiServiceException(HttpStatus.S_422_UNPROCESSABLE_ENTITY, "No configuration for the requested resource.");
    } catch (IOException ioe) {
        // Failed to close timer context. This should never happen
        throw new RuntimeException(ioe);
    }
}

Example 3 with NotConfiguredException

use of org.apache.gobblin.broker.iface.NotConfiguredException in project incubator-gobblin by apache.

the class PoliciesResource method get.

@Override
public Policy get(String resourceId) {
    try {
        ThrottlingPolicy throttlingPolicy = (ThrottlingPolicy) this.broker.getSharedResource(new ThrottlingPolicyFactory(), new SharedLimiterKey(resourceId));
        Policy restliPolicy = new Policy();
        restliPolicy.setPolicyName(throttlingPolicy.getClass().getSimpleName());
        restliPolicy.setResource(resourceId);
        restliPolicy.setParameters(new StringMap(throttlingPolicy.getParameters()));
        restliPolicy.setPolicyDetails(throttlingPolicy.getDescription());
        MetricContext resourceContext = (MetricContext) broker.getSharedResource(new MetricContextFactory(), new SubTaggedMetricContextKey(resourceId, ImmutableMap.of(RESOURCE_ID_TAG, resourceId)));
        StringMap metrics = new StringMap();
        for (Map.Entry<String, Meter> meter : resourceContext.getMeters().entrySet()) {
            metrics.put(meter.getKey(), Double.toString(meter.getValue().getOneMinuteRate()));
        }
        restliPolicy.setMetrics(metrics);
        return restliPolicy;
    } catch (NotConfiguredException nce) {
        throw new RestLiServiceException(HttpStatus.S_404_NOT_FOUND, "Policy not found for resource " + resourceId);
    }
}

Example 4 with NotConfiguredException

use of org.apache.gobblin.broker.iface.NotConfiguredException in project incubator-gobblin by apache.

the class StreamThrottler method doThrottleInputStream.

/**
 * Throttles an {@link InputStream} if throttling is configured.
 * @param inputStream {@link InputStream} to throttle.
 * @param sourceURI used for selecting the throttling policy.
 * @param targetURI used for selecting the throttling policy.
 */
@Builder(builderMethodName = "throttleInputStream", builderClassName = "InputStreamThrottler")
private ThrottledInputStream doThrottleInputStream(InputStream inputStream, URI sourceURI, URI targetURI) {
    Preconditions.checkNotNull(inputStream, "InputStream cannot be null.");
    Limiter limiter = new NoopLimiter();
    if (sourceURI != null && targetURI != null) {
        StreamCopierSharedLimiterKey key = new StreamCopierSharedLimiterKey(sourceURI, targetURI);
        try {
            limiter = new MultiLimiter(limiter, this.broker.getSharedResource(new SharedLimiterFactory<S>(), key));
        } catch (NotConfiguredException nce) {
            log.warn("Could not create a Limiter for key " + key, nce);
        }
    } else {
        log.info("Not throttling input stream because source or target URIs are not defined.");
    }
    Optional<MeteredInputStream> meteredStream = MeteredInputStream.findWrappedMeteredInputStream(inputStream);
    if (!meteredStream.isPresent()) {
        meteredStream = Optional.of(MeteredInputStream.builder().in(inputStream).build());
        inputStream = meteredStream.get();
    }
    return new ThrottledInputStream(inputStream, limiter, meteredStream.get());
}

Example 5 with NotConfiguredException

use of org.apache.gobblin.broker.iface.NotConfiguredException in project incubator-gobblin by apache.

the class SharedLimiterFactory method createResource.

@Override
public SharedResourceFactoryResponse<Limiter> createResource(SharedResourcesBroker<S> broker, ScopedConfigView<S, SharedLimiterKey> configView) throws NotConfiguredException {
    Config config = configView.getConfig();
    SharedLimiterKey.GlobalLimiterPolicy globalLimiterPolicy = configView.getKey().getGlobalLimiterPolicy();
    if (ConfigUtils.getBoolean(config, SKIP_GLOBAL_LIMITER_KEY, false)) {
        if (globalLimiterPolicy != SharedLimiterKey.GlobalLimiterPolicy.LOCAL_ONLY) {
            SharedLimiterKey modifiedKey = new SharedLimiterKey(configView.getKey().getResourceLimitedPath(), SharedLimiterKey.GlobalLimiterPolicy.LOCAL_ONLY);
            return new ResourceCoordinate<>(this, modifiedKey, (S) configView.getScope());
        }
    } else if (config.hasPath(FAIL_IF_NO_GLOBAL_LIMITER_KEY) && config.getBoolean(FAIL_IF_NO_GLOBAL_LIMITER_KEY) && globalLimiterPolicy != SharedLimiterKey.GlobalLimiterPolicy.USE_GLOBAL) {
        // if user has specified FAIL_IF_NO_GLOBAL_LIMITER_KEY, promote the policy from USE_GLOBAL_IF_CONFIGURED to USE_GLOBAL
        // e.g. fail if no GLOBAL configuration is present
        SharedLimiterKey modifiedKey = new SharedLimiterKey(configView.getKey().getResourceLimitedPath(), SharedLimiterKey.GlobalLimiterPolicy.USE_GLOBAL);
        return new ResourceCoordinate<>(this, modifiedKey, (S) configView.getScope());
    }
    Limiter limiter;
    if (!configView.getScope().isLocal() && !globalLimiterPolicy.equals(SharedLimiterKey.GlobalLimiterPolicy.LOCAL_ONLY)) {
        try {
            Class<?> klazz = Class.forName("org.apache.gobblin.util.limiter.RestliLimiterFactory");
            return new ResourceCoordinate<>((SharedResourceFactory<Limiter, SharedLimiterKey, S>) klazz.newInstance(), configView.getKey(), (S) configView.getScope());
        } catch (ReflectiveOperationException roe) {
            if (globalLimiterPolicy.equals(SharedLimiterKey.GlobalLimiterPolicy.USE_GLOBAL)) {
                throw new RuntimeException("There is no Global limiter factory in the classpath.");
            }
        }
    }
    if (config.hasPath(LIMITER_CLASS_KEY)) {
        try {
            LimiterFactory factory = RESOLVER.resolveClass(config.getString(LIMITER_CLASS_KEY)).newInstance();
            limiter = factory.buildLimiter(config);
        } catch (ReflectiveOperationException roe) {
            throw new RuntimeException(roe);
        }
    } else {
        if (config.hasPath(FAIL_ON_UNKNOWN_RESOURCE_ID) && config.getBoolean(FAIL_ON_UNKNOWN_RESOURCE_ID)) {
            throw new NotConfiguredException();
        }
        limiter = new NoopLimiter();
    }
    ScopeType<S> scope = configView.getScope();
    Collection<S> parentScopes = scope.parentScopes();
    if (parentScopes != null) {
        try {
            for (S parentScope : parentScopes) {
                limiter = new MultiLimiter(limiter, broker.getSharedResourceAtScope(this, configView.getKey(), parentScope));
            }
        } catch (NoSuchScopeException nsse) {
            throw new RuntimeException("Could not get higher scope limiter. This is an error in code.", nsse);
        }
    }
    return new ResourceInstance<>(limiter);
}