Examples with Authentication org.codelibs.fess.crawler.client.http.Authentication used on opensource projects

Example 1 with Authentication

use of org.codelibs.fess.crawler.client.http.Authentication in project fess by codelibs.

the class WebConfig method initializeClientFactory.

@Override
public Map<String, Object> initializeClientFactory(final CrawlerClientFactory clientFactory) {
    final WebAuthenticationService webAuthenticationService = ComponentUtil.getComponent(WebAuthenticationService.class);
    final RequestHeaderService requestHeaderService = ComponentUtil.getComponent(RequestHeaderService.class);
    final FessConfig fessConfig = ComponentUtil.getFessConfig();
    // HttpClient Parameters
    final Map<String, Object> paramMap = new HashMap<>();
    clientFactory.setInitParameterMap(paramMap);
    final Map<String, String> clientConfigMap = getConfigParameterMap(ConfigName.CLIENT);
    if (clientConfigMap != null) {
        paramMap.putAll(clientConfigMap);
    }
    // robots txt enabled
    if (paramMap.get(HcHttpClient.ROBOTS_TXT_ENABLED_PROPERTY) == null) {
        paramMap.put(HcHttpClient.ROBOTS_TXT_ENABLED_PROPERTY, !fessConfig.isCrawlerIgnoreRobotsTxt());
    }
    final String userAgent = getUserAgent();
    if (StringUtil.isNotBlank(userAgent)) {
        paramMap.put(HcHttpClient.USER_AGENT_PROPERTY, userAgent);
    }
    final List<WebAuthentication> webAuthList = webAuthenticationService.getWebAuthenticationList(getId());
    final List<Authentication> basicAuthList = new ArrayList<>();
    for (final WebAuthentication webAuth : webAuthList) {
        basicAuthList.add(webAuth.getAuthentication());
    }
    paramMap.put(HcHttpClient.BASIC_AUTHENTICATIONS_PROPERTY, basicAuthList.toArray(new Authentication[basicAuthList.size()]));
    // request header
    final List<RequestHeader> requestHeaderList = requestHeaderService.getRequestHeaderList(getId());
    final List<org.codelibs.fess.crawler.client.http.RequestHeader> rhList = new ArrayList<>();
    for (final RequestHeader requestHeader : requestHeaderList) {
        rhList.add(requestHeader.getCrawlerRequestHeader());
    }
    paramMap.put(HcHttpClient.REQUERT_HEADERS_PROPERTY, rhList.toArray(new org.codelibs.fess.crawler.client.http.RequestHeader[rhList.size()]));
    // proxy credentials
    if (paramMap.get("proxyUsername") != null && paramMap.get("proxyPassword") != null) {
        paramMap.put(HcHttpClient.PROXY_CREDENTIALS_PROPERTY, new UsernamePasswordCredentials(paramMap.remove("proxyUsername").toString(), paramMap.remove("proxyPassword").toString()));
    }
    return paramMap;
}

Example 2 with Authentication

use of org.codelibs.fess.crawler.client.http.Authentication in project fess by codelibs.

the class DataConfig method initializeClientFactory.

@Override
public Map<String, Object> initializeClientFactory(final CrawlerClientFactory crawlerClientFactory) {
    final Map<String, String> paramMap = getHandlerParameterMap();
    final Map<String, Object> factoryParamMap = new HashMap<>();
    crawlerClientFactory.setInitParameterMap(factoryParamMap);
    // parameters
    for (final Map.Entry<String, String> entry : paramMap.entrySet()) {
        final String key = entry.getKey();
        if (key.startsWith(CRAWLER_PARAM_PREFIX)) {
            factoryParamMap.put(key.substring(CRAWLER_PARAM_PREFIX.length()), entry.getValue());
        }
    }
    // user agent
    final String userAgent = paramMap.get(CRAWLER_USERAGENT);
    if (StringUtil.isNotBlank(userAgent)) {
        factoryParamMap.put(HcHttpClient.USER_AGENT_PROPERTY, userAgent);
    }
    // web auth
    final String webAuthStr = paramMap.get(CRAWLER_WEB_AUTH);
    if (StringUtil.isNotBlank(webAuthStr)) {
        final String[] webAuthNames = webAuthStr.split(",");
        final List<Authentication> basicAuthList = new ArrayList<>();
        for (final String webAuthName : webAuthNames) {
            final String scheme = paramMap.get(CRAWLER_WEB_AUTH + "." + webAuthName + ".scheme");
            final String hostname = paramMap.get(CRAWLER_WEB_AUTH + "." + webAuthName + ".host");
            final String port = paramMap.get(CRAWLER_WEB_AUTH + "." + webAuthName + ".port");
            final String realm = paramMap.get(CRAWLER_WEB_AUTH + "." + webAuthName + ".realm");
            final String username = paramMap.get(CRAWLER_WEB_AUTH + "." + webAuthName + ".username");
            final String password = paramMap.get(CRAWLER_WEB_AUTH + "." + webAuthName + ".password");
            if (StringUtil.isEmpty(username)) {
                logger.warn("username is empty. webAuth:" + webAuthName);
                continue;
            }
            AuthScheme authScheme = null;
            if (Constants.BASIC.equals(scheme)) {
                authScheme = new BasicScheme();
            } else if (Constants.DIGEST.equals(scheme)) {
                authScheme = new DigestScheme();
            } else if (Constants.NTLM.equals(scheme)) {
                authScheme = new NTLMScheme(new JcifsEngine());
            }
            // TODO FORM
            AuthScope authScope;
            if (StringUtil.isBlank(hostname)) {
                authScope = AuthScope.ANY;
            } else {
                int p = AuthScope.ANY_PORT;
                if (StringUtil.isNotBlank(port)) {
                    try {
                        p = Integer.parseInt(port);
                    } catch (final NumberFormatException e) {
                        logger.warn("Failed to parse " + port, e);
                    }
                }
                String r = realm;
                if (StringUtil.isBlank(realm)) {
                    r = AuthScope.ANY_REALM;
                }
                String s = scheme;
                if (StringUtil.isBlank(scheme) || Constants.NTLM.equals(scheme)) {
                    s = AuthScope.ANY_SCHEME;
                }
                authScope = new AuthScope(hostname, p, r, s);
            }
            Credentials credentials;
            if (Constants.NTLM.equals(scheme)) {
                final String workstation = paramMap.get(CRAWLER_WEB_AUTH + "." + webAuthName + ".workstation");
                final String domain = paramMap.get(CRAWLER_WEB_AUTH + "." + webAuthName + ".domain");
                credentials = new NTCredentials(username, password == null ? StringUtil.EMPTY : password, workstation == null ? StringUtil.EMPTY : workstation, domain == null ? StringUtil.EMPTY : domain);
            } else {
                credentials = new UsernamePasswordCredentials(username, password == null ? StringUtil.EMPTY : password);
            }
            basicAuthList.add(new AuthenticationImpl(authScope, credentials, authScheme));
        }
        factoryParamMap.put(HcHttpClient.BASIC_AUTHENTICATIONS_PROPERTY, basicAuthList.toArray(new Authentication[basicAuthList.size()]));
    }
    // request header
    final List<org.codelibs.fess.crawler.client.http.RequestHeader> rhList = new ArrayList<>();
    int count = 1;
    String headerName = paramMap.get(CRAWLER_WEB_HEADER_PREFIX + count + ".name");
    while (StringUtil.isNotBlank(headerName)) {
        final String headerValue = paramMap.get(CRAWLER_WEB_HEADER_PREFIX + count + ".value");
        rhList.add(new org.codelibs.fess.crawler.client.http.RequestHeader(headerName, headerValue));
        count++;
        headerName = paramMap.get(CRAWLER_WEB_HEADER_PREFIX + count + ".name");
    }
    if (!rhList.isEmpty()) {
        factoryParamMap.put(HcHttpClient.REQUERT_HEADERS_PROPERTY, rhList.toArray(new org.codelibs.fess.crawler.client.http.RequestHeader[rhList.size()]));
    }
    // file auth
    final String fileAuthStr = paramMap.get(CRAWLER_FILE_AUTH);
    if (StringUtil.isNotBlank(fileAuthStr)) {
        final String[] fileAuthNames = fileAuthStr.split(",");
        final List<SmbAuthentication> smbAuthList = new ArrayList<>();
        final List<FtpAuthentication> ftpAuthList = new ArrayList<>();
        for (final String fileAuthName : fileAuthNames) {
            final String scheme = paramMap.get(CRAWLER_FILE_AUTH + "." + fileAuthName + ".scheme");
            if (Constants.SAMBA.equals(scheme)) {
                final String domain = paramMap.get(CRAWLER_FILE_AUTH + "." + fileAuthName + ".domain");
                final String hostname = paramMap.get(CRAWLER_FILE_AUTH + "." + fileAuthName + ".host");
                final String port = paramMap.get(CRAWLER_FILE_AUTH + "." + fileAuthName + ".port");
                final String username = paramMap.get(CRAWLER_FILE_AUTH + "." + fileAuthName + ".username");
                final String password = paramMap.get(CRAWLER_FILE_AUTH + "." + fileAuthName + ".password");
                if (StringUtil.isEmpty(username)) {
                    logger.warn("username is empty. fileAuth:" + fileAuthName);
                    continue;
                }
                final SmbAuthentication smbAuth = new SmbAuthentication();
                smbAuth.setDomain(domain == null ? StringUtil.EMPTY : domain);
                smbAuth.setServer(hostname);
                if (StringUtil.isNotBlank(port)) {
                    try {
                        smbAuth.setPort(Integer.parseInt(port));
                    } catch (final NumberFormatException e) {
                        logger.warn("Failed to parse " + port, e);
                    }
                }
                smbAuth.setUsername(username);
                smbAuth.setPassword(password == null ? StringUtil.EMPTY : password);
                smbAuthList.add(smbAuth);
            } else if (Constants.FTP.equals(scheme)) {
                final String hostname = paramMap.get(CRAWLER_FILE_AUTH + "." + fileAuthName + ".host");
                final String port = paramMap.get(CRAWLER_FILE_AUTH + "." + fileAuthName + ".port");
                final String username = paramMap.get(CRAWLER_FILE_AUTH + "." + fileAuthName + ".username");
                final String password = paramMap.get(CRAWLER_FILE_AUTH + "." + fileAuthName + ".password");
                if (StringUtil.isEmpty(username)) {
                    logger.warn("username is empty. fileAuth:" + fileAuthName);
                    continue;
                }
                final FtpAuthentication ftpAuth = new FtpAuthentication();
                ftpAuth.setServer(hostname);
                if (StringUtil.isNotBlank(port)) {
                    try {
                        ftpAuth.setPort(Integer.parseInt(port));
                    } catch (final NumberFormatException e) {
                        logger.warn("Failed to parse " + port, e);
                    }
                }
                ftpAuth.setUsername(username);
                ftpAuth.setPassword(password == null ? StringUtil.EMPTY : password);
                ftpAuthList.add(ftpAuth);
            }
        }
        if (!smbAuthList.isEmpty()) {
            factoryParamMap.put(SmbClient.SMB_AUTHENTICATIONS_PROPERTY, smbAuthList.toArray(new SmbAuthentication[smbAuthList.size()]));
        }
        if (!ftpAuthList.isEmpty()) {
            factoryParamMap.put(FtpClient.FTP_AUTHENTICATIONS_PROPERTY, ftpAuthList.toArray(new FtpAuthentication[ftpAuthList.size()]));
        }
    }
    return factoryParamMap;
}

Example 3 with Authentication

use of org.codelibs.fess.crawler.client.http.Authentication in project fess-crawler by codelibs.

the class ApiExtractor method init.

@PostConstruct
public void init() {
    if (logger.isDebugEnabled()) {
        logger.debug("Initializing " + ApiExtractor.class.getName());
    }
    // httpclient
    final org.apache.http.client.config.RequestConfig.Builder requestConfigBuilder = RequestConfig.custom();
    final HttpClientBuilder httpClientBuilder = HttpClientBuilder.create();
    final Integer connectionTimeoutParam = connectionTimeout;
    if (connectionTimeoutParam != null) {
        requestConfigBuilder.setConnectTimeout(connectionTimeoutParam);
    }
    final Integer soTimeoutParam = soTimeout;
    if (soTimeoutParam != null) {
        requestConfigBuilder.setSocketTimeout(soTimeoutParam);
    }
    // AuthSchemeFactory
    final RegistryBuilder<AuthSchemeProvider> authSchemeProviderBuilder = RegistryBuilder.create();
    // @SuppressWarnings("unchecked")
    final Map<String, AuthSchemeProvider> factoryMap = authSchemeProviderMap;
    if (factoryMap != null) {
        for (final Map.Entry<String, AuthSchemeProvider> entry : factoryMap.entrySet()) {
            authSchemeProviderBuilder.register(entry.getKey(), entry.getValue());
        }
    }
    // user agent
    if (StringUtil.isNotBlank(userAgent)) {
        httpClientBuilder.setUserAgent(userAgent);
    }
    // Authentication
    final Authentication[] siteCredentialList = new Authentication[0];
    for (final Authentication authentication : siteCredentialList) {
        final AuthScope authScope = authentication.getAuthScope();
        credentialsProvider.setCredentials(authScope, authentication.getCredentials());
        final AuthScheme authScheme = authentication.getAuthScheme();
        if (authScope.getHost() != null && authScheme != null) {
            final HttpHost targetHost = new HttpHost(authScope.getHost(), authScope.getPort());
            authCache.put(targetHost, authScheme);
        }
    }
    httpClientContext.setAuthCache(authCache);
    httpClientContext.setCredentialsProvider(credentialsProvider);
    // Request Header
    final RequestHeader[] requestHeaders = { new RequestHeader("enctype", "multipart/form-data") };
    for (final RequestHeader requestHeader : requestHeaders) {
        if (requestHeader.isValid()) {
            requestHeaderList.add(new BasicHeader(requestHeader.getName(), requestHeader.getValue()));
        }
    }
    final CloseableHttpClient closeableHttpClient = httpClientBuilder.setDefaultRequestConfig(requestConfigBuilder.build()).build();
    if (!httpClientPropertyMap.isEmpty()) {
        final BeanDesc beanDesc = BeanDescFactory.getBeanDesc(closeableHttpClient.getClass());
        for (final Map.Entry<String, Object> entry : httpClientPropertyMap.entrySet()) {
            final String propertyName = entry.getKey();
            if (beanDesc.hasPropertyDesc(propertyName)) {
                final PropertyDesc propertyDesc = beanDesc.getPropertyDesc(propertyName);
                propertyDesc.setValue(closeableHttpClient, entry.getValue());
            } else {
                logger.warn("DefaultHttpClient does not have " + propertyName + ".");
            }
        }
    }
    httpClient = closeableHttpClient;
}

Example 4 with Authentication

use of org.codelibs.fess.crawler.client.http.Authentication in project fess by codelibs.

the class DataConfig method initializeClientFactory.

@Override
public CrawlerClientFactory initializeClientFactory(final Supplier<CrawlerClientFactory> creator) {
    if (crawlerClientFactory != null) {
        return crawlerClientFactory;
    }
    final CrawlerClientFactory factory = creator.get();
    final Map<String, String> paramMap = getHandlerParameterMap();
    final Map<String, Object> factoryParamMap = new HashMap<>();
    factory.setInitParameterMap(factoryParamMap);
    // parameters
    for (final Map.Entry<String, String> entry : paramMap.entrySet()) {
        final String key = entry.getKey();
        if (key.startsWith(CRAWLER_PARAM_PREFIX)) {
            factoryParamMap.put(key.substring(CRAWLER_PARAM_PREFIX.length()), entry.getValue());
        }
    }
    // user agent
    final String userAgent = paramMap.get(CRAWLER_USERAGENT);
    if (StringUtil.isNotBlank(userAgent)) {
        factoryParamMap.put(HcHttpClient.USER_AGENT_PROPERTY, userAgent);
    }
    // web auth
    final String webAuthStr = paramMap.get(CRAWLER_WEB_AUTH);
    if (StringUtil.isNotBlank(webAuthStr)) {
        final String[] webAuthNames = webAuthStr.split(",");
        final List<Authentication> basicAuthList = new ArrayList<>();
        for (final String webAuthName : webAuthNames) {
            final String scheme = paramMap.get(CRAWLER_WEB_AUTH + "." + webAuthName + ".scheme");
            final AuthScheme authScheme = getAuthScheme(paramMap, webAuthName, scheme);
            final AuthScope authScope = getAuthScope(webAuthName, scheme, paramMap);
            final Credentials credentials = getCredentials(webAuthName, scheme, paramMap);
            basicAuthList.add(new AuthenticationImpl(authScope, credentials, authScheme));
        }
        factoryParamMap.put(HcHttpClient.BASIC_AUTHENTICATIONS_PROPERTY, basicAuthList.toArray(new Authentication[basicAuthList.size()]));
    }
    // request header
    final List<org.codelibs.fess.crawler.client.http.RequestHeader> rhList = new ArrayList<>();
    int count = 1;
    String headerName = paramMap.get(CRAWLER_WEB_HEADER_PREFIX + count + ".name");
    while (StringUtil.isNotBlank(headerName)) {
        final String headerValue = paramMap.get(CRAWLER_WEB_HEADER_PREFIX + count + ".value");
        rhList.add(new org.codelibs.fess.crawler.client.http.RequestHeader(headerName, headerValue));
        count++;
        headerName = paramMap.get(CRAWLER_WEB_HEADER_PREFIX + count + ".name");
    }
    if (!rhList.isEmpty()) {
        factoryParamMap.put(HcHttpClient.REQUERT_HEADERS_PROPERTY, rhList.toArray(new org.codelibs.fess.crawler.client.http.RequestHeader[rhList.size()]));
    }
    // proxy credentials
    final String proxyHost = paramMap.get(CRAWLER_WEB_PREFIX + "proxyHost");
    final String proxyPort = paramMap.get(CRAWLER_WEB_PREFIX + "proxyPort");
    if (StringUtil.isNotBlank(proxyHost) && StringUtil.isNotBlank(proxyPort)) {
        factoryParamMap.put(HcHttpClient.PROXY_HOST_PROPERTY, proxyHost);
        factoryParamMap.put(HcHttpClient.PROXY_PORT_PROPERTY, proxyPort);
        final String proxyUsername = paramMap.get(CRAWLER_WEB_PREFIX + "proxyUsername");
        final String proxyPassword = paramMap.get(CRAWLER_WEB_PREFIX + "proxyPassword");
        if (proxyUsername != null && proxyPassword != null) {
            factoryParamMap.put(HcHttpClient.PROXY_CREDENTIALS_PROPERTY, new UsernamePasswordCredentials(proxyUsername, proxyPassword));
        }
    } else {
        initializeDefaultHttpProxy(factoryParamMap);
    }
    // file auth
    final String fileAuthStr = paramMap.get(CRAWLER_FILE_AUTH);
    if (StringUtil.isNotBlank(fileAuthStr)) {
        final String[] fileAuthNames = fileAuthStr.split(",");
        final List<SmbAuthentication> smbAuthList = new ArrayList<>();
        final List<org.codelibs.fess.crawler.client.smb1.SmbAuthentication> smb1AuthList = new ArrayList<>();
        final List<FtpAuthentication> ftpAuthList = new ArrayList<>();
        for (final String fileAuthName : fileAuthNames) {
            final String scheme = paramMap.get(CRAWLER_FILE_AUTH + "." + fileAuthName + ".scheme");
            if (Constants.SAMBA.equals(scheme)) {
                final String domain = paramMap.get(CRAWLER_FILE_AUTH + "." + fileAuthName + ".domain");
                final String hostname = paramMap.get(CRAWLER_FILE_AUTH + "." + fileAuthName + ".host");
                final String port = paramMap.get(CRAWLER_FILE_AUTH + "." + fileAuthName + ".port");
                final String username = paramMap.get(CRAWLER_FILE_AUTH + "." + fileAuthName + ".username");
                final String password = paramMap.get(CRAWLER_FILE_AUTH + "." + fileAuthName + ".password");
                if (StringUtil.isEmpty(username)) {
                    logger.warn("username is empty. fileAuth:{}", fileAuthName);
                    continue;
                }
                final SmbAuthentication smbAuth = new SmbAuthentication();
                smbAuth.setDomain(domain == null ? StringUtil.EMPTY : domain);
                smbAuth.setServer(hostname);
                if (StringUtil.isNotBlank(port)) {
                    try {
                        smbAuth.setPort(Integer.parseInt(port));
                    } catch (final NumberFormatException e) {
                        logger.warn("Failed to parse {}", port, e);
                    }
                }
                smbAuth.setUsername(username);
                smbAuth.setPassword(password == null ? StringUtil.EMPTY : password);
                smbAuthList.add(smbAuth);
                final org.codelibs.fess.crawler.client.smb1.SmbAuthentication smb1Auth = new org.codelibs.fess.crawler.client.smb1.SmbAuthentication();
                smb1Auth.setDomain(domain == null ? StringUtil.EMPTY : domain);
                smb1Auth.setServer(hostname);
                if (StringUtil.isNotBlank(port)) {
                    try {
                        smb1Auth.setPort(Integer.parseInt(port));
                    } catch (final NumberFormatException e) {
                        logger.warn("Failed to parse {}", port, e);
                    }
                }
                smb1Auth.setUsername(username);
                smb1Auth.setPassword(password == null ? StringUtil.EMPTY : password);
                smb1AuthList.add(smb1Auth);
            } else if (Constants.FTP.equals(scheme)) {
                final String hostname = paramMap.get(CRAWLER_FILE_AUTH + "." + fileAuthName + ".host");
                final String port = paramMap.get(CRAWLER_FILE_AUTH + "." + fileAuthName + ".port");
                final String username = paramMap.get(CRAWLER_FILE_AUTH + "." + fileAuthName + ".username");
                final String password = paramMap.get(CRAWLER_FILE_AUTH + "." + fileAuthName + ".password");
                if (StringUtil.isEmpty(username)) {
                    logger.warn("username is empty. fileAuth:{}", fileAuthName);
                    continue;
                }
                final FtpAuthentication ftpAuth = new FtpAuthentication();
                ftpAuth.setServer(hostname);
                if (StringUtil.isNotBlank(port)) {
                    try {
                        ftpAuth.setPort(Integer.parseInt(port));
                    } catch (final NumberFormatException e) {
                        logger.warn("Failed to parse {}", port, e);
                    }
                }
                ftpAuth.setUsername(username);
                ftpAuth.setPassword(password == null ? StringUtil.EMPTY : password);
                ftpAuthList.add(ftpAuth);
            }
        }
        if (!smbAuthList.isEmpty()) {
            factoryParamMap.put(SmbClient.SMB_AUTHENTICATIONS_PROPERTY, smbAuthList.toArray(new SmbAuthentication[smbAuthList.size()]));
        }
        if (!smb1AuthList.isEmpty()) {
            factoryParamMap.put(org.codelibs.fess.crawler.client.smb1.SmbClient.SMB_AUTHENTICATIONS_PROPERTY, smb1AuthList.toArray(new org.codelibs.fess.crawler.client.smb1.SmbAuthentication[smb1AuthList.size()]));
        }
        if (!ftpAuthList.isEmpty()) {
            factoryParamMap.put(FtpClient.FTP_AUTHENTICATIONS_PROPERTY, ftpAuthList.toArray(new FtpAuthentication[ftpAuthList.size()]));
        }
    }
    crawlerClientFactory = factory;
    return factory;
}

Example 5 with Authentication

use of org.codelibs.fess.crawler.client.http.Authentication in project fess by codelibs.

the class WebConfig method initializeClientFactory.

@Override
public CrawlerClientFactory initializeClientFactory(final Supplier<CrawlerClientFactory> creator) {
    if (crawlerClientFactory != null) {
        return crawlerClientFactory;
    }
    final CrawlerClientFactory factory = creator.get();
    final WebAuthenticationService webAuthenticationService = ComponentUtil.getComponent(WebAuthenticationService.class);
    final RequestHeaderService requestHeaderService = ComponentUtil.getComponent(RequestHeaderService.class);
    final FessConfig fessConfig = ComponentUtil.getFessConfig();
    // HttpClient Parameters
    final Map<String, Object> paramMap = new HashMap<>();
    factory.setInitParameterMap(paramMap);
    final Map<String, String> clientConfigMap = getConfigParameterMap(ConfigName.CLIENT);
    if (clientConfigMap != null) {
        paramMap.putAll(clientConfigMap);
    }
    // robots txt enabled
    if (paramMap.get(Param.Client.ROBOTS_TXT_ENABLED) == null) {
        paramMap.put(Param.Client.ROBOTS_TXT_ENABLED, !fessConfig.isCrawlerIgnoreRobotsTxt());
    }
    final String userAgent = getUserAgent();
    if (StringUtil.isNotBlank(userAgent)) {
        paramMap.put(Client.USER_AGENT, userAgent);
    }
    final List<WebAuthentication> webAuthList = webAuthenticationService.getWebAuthenticationList(getId());
    final List<Authentication> basicAuthList = new ArrayList<>();
    for (final WebAuthentication webAuth : webAuthList) {
        basicAuthList.add(webAuth.getAuthentication());
    }
    paramMap.put(HcHttpClient.BASIC_AUTHENTICATIONS_PROPERTY, basicAuthList.toArray(new Authentication[basicAuthList.size()]));
    // request header
    final List<RequestHeader> requestHeaderList = requestHeaderService.getRequestHeaderList(getId());
    final List<org.codelibs.fess.crawler.client.http.RequestHeader> rhList = new ArrayList<>();
    for (final RequestHeader requestHeader : requestHeaderList) {
        rhList.add(requestHeader.getCrawlerRequestHeader());
    }
    paramMap.put(HcHttpClient.REQUERT_HEADERS_PROPERTY, rhList.toArray(new org.codelibs.fess.crawler.client.http.RequestHeader[rhList.size()]));
    final String proxyHost = (String) paramMap.get(Param.Client.PROXY_HOST);
    final String proxyPort = (String) paramMap.get(Param.Client.PROXY_PORT);
    if (StringUtil.isNotBlank(proxyHost) && StringUtil.isNotBlank(proxyPort)) {
        // proxy credentials
        if (paramMap.get(Param.Client.PROXY_USERNAME) != null && paramMap.get(Param.Client.PROXY_PASSWORD) != null) {
            paramMap.put(HcHttpClient.PROXY_CREDENTIALS_PROPERTY, new UsernamePasswordCredentials(paramMap.remove(Param.Client.PROXY_USERNAME).toString(), paramMap.remove(Param.Client.PROXY_PASSWORD).toString()));
        }
    } else {
        initializeDefaultHttpProxy(paramMap);
    }
    crawlerClientFactory = factory;
    return factory;
}