Examples with COSCredentialProviderList org.apache.hadoop.fs.auth.COSCredentialProviderList used on opensource projects

Example 1 with COSCredentialProviderList

use of org.apache.hadoop.fs.auth.COSCredentialProviderList in project hadoop-cos by tencentyun.

the class CosNativeFileSystemStore method initCOSClient.

private void initCOSClient(URI uri, Configuration conf) throws IOException {
    COSCredentialProviderList cosCredentialProviderList = CosNUtils.createCosCredentialsProviderSet(uri, conf);
    ClientConfig config;
    String customerDomain = conf.get(CosNConfigKeys.CUSTOMER_DOMAIN);
    if (null == customerDomain) {
        String region = conf.get(CosNConfigKeys.COSN_REGION_KEY);
        if (null == region) {
            region = conf.get(CosNConfigKeys.COSN_REGION_PREV_KEY);
        }
        String endpointSuffix = conf.get(CosNConfigKeys.COSN_ENDPOINT_SUFFIX_KEY);
        if (null == endpointSuffix) {
            endpointSuffix = conf.get(CosNConfigKeys.COSN_ENDPOINT_SUFFIX_PREV_KEY);
        }
        if (null == region && null == endpointSuffix) {
            String exceptionMsg = String.format("config '%s' and '%s' specify at least one.", CosNConfigKeys.COSN_REGION_KEY, CosNConfigKeys.COSN_ENDPOINT_SUFFIX_KEY);
            throw new IOException(exceptionMsg);
        }
        if (null == region) {
            config = new ClientConfig(new Region(""));
            config.setEndPointSuffix(endpointSuffix);
        } else {
            config = new ClientConfig(new Region(region));
        }
        this.useL5Id = conf.getBoolean(CosNConfigKeys.COSN_USE_L5_ENABLE, CosNConfigKeys.DEFAULT_COSN_USE_L5_ENABLE);
        this.l5UpdateMaxRetryTimes = conf.getInt(CosNConfigKeys.COSN_L5_UPDATE_MAX_RETRIES_KEY, CosNConfigKeys.DEFAULT_COSN_L5_UPDATE_MAX_RETRIES);
        if (useL5Id) {
            String l5Id = conf.get(CosNConfigKeys.COSN_L5_KEY);
            if (null != l5Id) {
                int l5modId = Integer.parseInt(l5Id.split(",")[0]);
                int l5cmdId = Integer.parseInt(l5Id.split(",")[1]);
                Class<?> l5EndpointResolverClass;
                try {
                    l5EndpointResolverClass = Class.forName("org.apache.hadoop.fs.cosn.TencentCloudL5EndpointResolverImpl");
                    this.l5EndpointResolver = (TencentCloudL5EndpointResolver) l5EndpointResolverClass.newInstance();
                    this.l5EndpointResolver.setModId(l5modId);
                    this.l5EndpointResolver.setCmdId(l5cmdId);
                } catch (ClassNotFoundException | InstantiationException | IllegalAccessException e) {
                    throw new IOException("The current version does not support L5 resolver.", e);
                }
                config.setEndpointResolver(l5EndpointResolver);
            }
        }
    } else {
        config = new ClientConfig(new Region(""));
        LOG.info("Use Customer Domain is {}", customerDomain);
        CustomerDomainEndpointResolver customerDomainEndpointResolver = new CustomerDomainEndpointResolver();
        customerDomainEndpointResolver.setEndpoint(customerDomain);
        config.setEndpointBuilder(customerDomainEndpointResolver);
    }
    boolean useHttps = conf.getBoolean(CosNConfigKeys.COSN_USE_HTTPS_KEY, CosNConfigKeys.DEFAULT_USE_HTTPS);
    if (useHttps) {
        config.setHttpProtocol(HttpProtocol.https);
    } else {
        // 这里 config 的默认值改过了，默认值变成了https了。
        config.setHttpProtocol(HttpProtocol.http);
    }
    int socketTimeoutSec = conf.getInt(CosNConfigKeys.COSN_CLIENT_SOCKET_TIMEOUTSEC, CosNConfigKeys.DEFAULT_CLIENT_SOCKET_TIMEOUTSEC);
    config.setSocketTimeout(socketTimeoutSec * 1000);
    this.crc32cEnabled = conf.getBoolean(CosNConfigKeys.CRC32C_CHECKSUM_ENABLED, CosNConfigKeys.DEFAULT_CRC32C_CHECKSUM_ENABLED);
    this.completeMPUCheckEnabled = conf.getBoolean(CosNConfigKeys.COSN_COMPLETE_MPU_CHECK, CosNConfigKeys.DEFAULT_COSN_COMPLETE_MPU_CHECK_ENABLE);
    // Proxy settings
    String httpProxyIp = conf.getTrimmed(CosNConfigKeys.HTTP_PROXY_IP);
    int httpProxyPort = conf.getInt(CosNConfigKeys.HTTP_PROXY_PORT, CosNConfigKeys.DEFAULT_HTTP_PROXY_PORT);
    if (null != httpProxyIp && !httpProxyIp.isEmpty()) {
        config.setHttpProxyIp(httpProxyIp);
        if (httpProxyPort >= 0) {
            config.setHttpProxyPort(httpProxyPort);
        } else {
            if (useHttps) {
                LOG.warn("Proxy IP set without port. Using HTTPS default port 443");
                config.setHttpProxyPort(443);
            } else {
                LOG.warn("Proxy IP set without port, Using HTTP default port 80");
                config.setHttpProxyPort(80);
            }
        }
        // setting the proxy username and password
        String proxyUserName = conf.get(CosNConfigKeys.HTTP_PROXY_USERNAME);
        String proxyPassword = conf.get(CosNConfigKeys.HTTP_PROXY_PASSWORD);
        if ((null == proxyUserName || proxyUserName.isEmpty()) != (null == proxyPassword || proxyPassword.isEmpty())) {
            String exceptionMessage = String.format("Proxy error: '%s' or '%s' set without the other.", CosNConfigKeys.HTTP_PROXY_USERNAME, CosNConfigKeys.HTTP_PROXY_PASSWORD);
            throw new IllegalArgumentException(exceptionMessage);
        }
        config.setProxyUsername(proxyUserName);
        config.setProxyPassword(proxyPassword);
    }
    String versionNum = getPluginVersionInfo();
    String versionInfo = versionNum.equals("unknown") ? CosNConfigKeys.DEFAULT_USER_AGENT : "cos-hadoop-plugin-v" + versionNum;
    String userAgent = conf.get(CosNConfigKeys.USER_AGENT, versionInfo);
    String emrVersion = conf.get(CosNConfigKeys.TENCENT_EMR_VERSION_KEY);
    if (null != emrVersion && !emrVersion.isEmpty()) {
        userAgent = String.format("%s on %s", userAgent, emrVersion);
    }
    config.setUserAgent(userAgent);
    this.maxRetryTimes = conf.getInt(CosNConfigKeys.COSN_MAX_RETRIES_KEY, CosNConfigKeys.DEFAULT_MAX_RETRIES);
    // 设置COSClient的最大重试次数
    int clientMaxRetryTimes = conf.getInt(CosNConfigKeys.CLIENT_MAX_RETRIES_KEY, CosNConfigKeys.DEFAULT_CLIENT_MAX_RETRIES);
    config.setMaxErrorRetry(clientMaxRetryTimes);
    LOG.info("hadoop cos retry times: {}, cos client retry times: {}", this.maxRetryTimes, clientMaxRetryTimes);
    // 设置连接池的最大连接数目
    config.setMaxConnectionsCount(conf.getInt(CosNConfigKeys.MAX_CONNECTION_NUM, CosNConfigKeys.DEFAULT_MAX_CONNECTION_NUM));
    // 设置是否进行服务器端加密
    String serverSideEncryptionAlgorithm = conf.get(CosNConfigKeys.COSN_SERVER_SIDE_ENCRYPTION_ALGORITHM, "");
    CosNEncryptionMethods cosSSE = CosNEncryptionMethods.getMethod(serverSideEncryptionAlgorithm);
    String sseKey = conf.get(CosNConfigKeys.COSN_SERVER_SIDE_ENCRYPTION_KEY, "");
    String sseContext = conf.get(CosNConfigKeys.COSN_SERVER_SIDE_ENCRYPTION_CONTEXT, "");
    checkEncryptionMethod(config, cosSSE, sseKey);
    this.encryptionSecrets = new CosNEncryptionSecrets(cosSSE, sseKey, sseContext);
    // Set the traffic limit
    this.trafficLimit = conf.getInt(CosNConfigKeys.TRAFFIC_LIMIT, CosNConfigKeys.DEFAULT_TRAFFIC_LIMIT);
    if (this.trafficLimit >= 0 && (this.trafficLimit < 100 * 1024 * 8 || this.trafficLimit > 100 * 1024 * 1024 * 8)) {
        String exceptionMessage = String.format("The '%s' needs to be between %d and %d. but %d.", CosNConfigKeys.TRAFFIC_LIMIT, 100 * 1024 * 8, 100 * 1024 * 1024 * 8, this.trafficLimit);
        throw new IllegalArgumentException(exceptionMessage);
    }
    this.cosClient = new COSClient(cosCredentialProviderList, config);
}