use of org.apache.hadoop.io.retry.RetryPolicy in project hadoop by apache.
the class NMProxy method createNMProxy.
public static <T> T createNMProxy(final Configuration conf, final Class<T> protocol, final UserGroupInformation ugi, final YarnRPC rpc, final InetSocketAddress serverAddress) {
RetryPolicy retryPolicy = createRetryPolicy(conf, YarnConfiguration.CLIENT_NM_CONNECT_MAX_WAIT_MS, YarnConfiguration.DEFAULT_CLIENT_NM_CONNECT_MAX_WAIT_MS, YarnConfiguration.CLIENT_NM_CONNECT_RETRY_INTERVAL_MS, YarnConfiguration.DEFAULT_CLIENT_NM_CONNECT_RETRY_INTERVAL_MS);
Configuration confClone = new Configuration(conf);
confClone.setInt(CommonConfigurationKeysPublic.IPC_CLIENT_CONNECT_MAX_RETRIES_KEY, 0);
confClone.setInt(CommonConfigurationKeysPublic.IPC_CLIENT_CONNECT_MAX_RETRIES_ON_SOCKET_TIMEOUTS_KEY, 0);
return createRetriableProxy(confClone, protocol, ugi, rpc, serverAddress, retryPolicy);
}
use of org.apache.hadoop.io.retry.RetryPolicy in project hadoop by apache.
the class RMProxy method createRMProxy.
/**
* @deprecated
* This method is deprecated and is not used by YARN internally any more.
* To create a proxy to the RM, use ClientRMProxy#createRMProxy or
* ServerRMProxy#createRMProxy.
*
* Create a proxy to the ResourceManager at the specified address.
*
* @param conf Configuration to generate retry policy
* @param protocol Protocol for the proxy
* @param rmAddress Address of the ResourceManager
* @param <T> Type information of the proxy
* @return Proxy to the RM
* @throws IOException
*/
@Deprecated
public static <T> T createRMProxy(final Configuration conf, final Class<T> protocol, InetSocketAddress rmAddress) throws IOException {
RetryPolicy retryPolicy = createRetryPolicy(conf, HAUtil.isHAEnabled(conf));
T proxy = RMProxy.<T>getProxy(conf, protocol, rmAddress);
LOG.info("Connecting to ResourceManager at " + rmAddress);
return (T) RetryProxy.create(protocol, proxy, retryPolicy);
}
use of org.apache.hadoop.io.retry.RetryPolicy in project hadoop by apache.
the class RMProxy method createRetryPolicy.
/**
* Fetch retry policy from Configuration and create the
* retry policy with specified retryTime and retry interval.
*/
protected static RetryPolicy createRetryPolicy(Configuration conf, long retryTime, long retryInterval, boolean isHAEnabled) {
long rmConnectWaitMS = retryTime;
long rmConnectionRetryIntervalMS = retryInterval;
boolean waitForEver = (rmConnectWaitMS == -1);
if (!waitForEver) {
if (rmConnectWaitMS < 0) {
throw new YarnRuntimeException("Invalid Configuration. " + YarnConfiguration.RESOURCEMANAGER_CONNECT_MAX_WAIT_MS + " can be -1, but can not be other negative numbers");
}
// try connect once
if (rmConnectWaitMS < rmConnectionRetryIntervalMS) {
LOG.warn(YarnConfiguration.RESOURCEMANAGER_CONNECT_MAX_WAIT_MS + " is smaller than " + YarnConfiguration.RESOURCEMANAGER_CONNECT_RETRY_INTERVAL_MS + ". Only try connect once.");
rmConnectWaitMS = 0;
}
}
// Handle HA case first
if (isHAEnabled) {
final long failoverSleepBaseMs = conf.getLong(YarnConfiguration.CLIENT_FAILOVER_SLEEPTIME_BASE_MS, rmConnectionRetryIntervalMS);
final long failoverSleepMaxMs = conf.getLong(YarnConfiguration.CLIENT_FAILOVER_SLEEPTIME_MAX_MS, rmConnectionRetryIntervalMS);
int maxFailoverAttempts = conf.getInt(YarnConfiguration.CLIENT_FAILOVER_MAX_ATTEMPTS, -1);
if (maxFailoverAttempts == -1) {
if (waitForEver) {
maxFailoverAttempts = Integer.MAX_VALUE;
} else {
maxFailoverAttempts = (int) (rmConnectWaitMS / failoverSleepBaseMs);
}
}
return RetryPolicies.failoverOnNetworkException(RetryPolicies.TRY_ONCE_THEN_FAIL, maxFailoverAttempts, failoverSleepBaseMs, failoverSleepMaxMs);
}
if (rmConnectionRetryIntervalMS < 0) {
throw new YarnRuntimeException("Invalid Configuration. " + YarnConfiguration.RESOURCEMANAGER_CONNECT_RETRY_INTERVAL_MS + " should not be negative.");
}
RetryPolicy retryPolicy = null;
if (waitForEver) {
retryPolicy = RetryPolicies.retryForeverWithFixedSleep(rmConnectionRetryIntervalMS, TimeUnit.MILLISECONDS);
} else {
retryPolicy = RetryPolicies.retryUpToMaximumTimeWithFixedSleep(rmConnectWaitMS, rmConnectionRetryIntervalMS, TimeUnit.MILLISECONDS);
}
Map<Class<? extends Exception>, RetryPolicy> exceptionToPolicyMap = new HashMap<Class<? extends Exception>, RetryPolicy>();
exceptionToPolicyMap.put(EOFException.class, retryPolicy);
exceptionToPolicyMap.put(ConnectException.class, retryPolicy);
exceptionToPolicyMap.put(NoRouteToHostException.class, retryPolicy);
exceptionToPolicyMap.put(UnknownHostException.class, retryPolicy);
exceptionToPolicyMap.put(ConnectTimeoutException.class, retryPolicy);
exceptionToPolicyMap.put(RetriableException.class, retryPolicy);
exceptionToPolicyMap.put(SocketException.class, retryPolicy);
exceptionToPolicyMap.put(StandbyException.class, retryPolicy);
// YARN-4288: local IOException is also possible.
exceptionToPolicyMap.put(IOException.class, retryPolicy);
// Not retry on remote IO exception.
return RetryPolicies.retryOtherThanRemoteException(RetryPolicies.TRY_ONCE_THEN_FAIL, exceptionToPolicyMap);
}
use of org.apache.hadoop.io.retry.RetryPolicy in project hadoop by apache.
the class RMProxy method createRMProxy.
/**
* Currently, used by NodeManagers only.
* Create a proxy for the specified protocol. For non-HA,
* this is a direct connection to the ResourceManager address. When HA is
* enabled, the proxy handles the failover between the ResourceManagers as
* well.
*/
@Private
protected static <T> T createRMProxy(final Configuration configuration, final Class<T> protocol, RMProxy instance, final long retryTime, final long retryInterval) throws IOException {
YarnConfiguration conf = (configuration instanceof YarnConfiguration) ? (YarnConfiguration) configuration : new YarnConfiguration(configuration);
RetryPolicy retryPolicy = createRetryPolicy(conf, retryTime, retryInterval, HAUtil.isHAEnabled(conf));
return newProxyInstance(conf, protocol, instance, retryPolicy);
}
use of org.apache.hadoop.io.retry.RetryPolicy in project hadoop by apache.
the class RequestHedgingRMFailoverProxyProvider method createRetriableProxy.
@SuppressWarnings("unchecked")
protected T createRetriableProxy() {
try {
// Create proxy that can retry exceptions properly.
RetryPolicy retryPolicy = RMProxy.createRetryPolicy(conf, false);
InetSocketAddress rmAddress = rmProxy.getRMAddress(conf, protocol);
T proxy = RMProxy.<T>getProxy(conf, protocol, rmAddress);
return (T) RetryProxy.create(protocol, proxy, retryPolicy);
} catch (IOException ioe) {
LOG.error("Unable to create proxy to the ResourceManager " + HAUtil.getRMHAId(conf), ioe);
return null;
}
}
Aggregations