Examples with AuthScheme org.apache.http.auth.AuthScheme used on opensource projects

Example 26 with AuthScheme

use of org.apache.http.auth.AuthScheme in project ecf by eclipse.

the class NTLMProxyDetector method isProxyType.

private static boolean isProxyType(HttpContext context, String scheme) {
    if (context == null)
        return false;
    AuthState authState = (AuthState) context.getAttribute(ClientContext.PROXY_AUTH_STATE);
    if (authState == null)
        return false;
    AuthScheme authScheme = authState.getAuthScheme();
    if (authScheme == null)
        return false;
    String schemeName = authScheme.getSchemeName();
    if (schemeName == null)
        return false;
    return schemeName.equalsIgnoreCase(scheme);
}

Example 27 with AuthScheme

use of org.apache.http.auth.AuthScheme in project fess-crawler by codelibs.

the class HcHttpClient method init.

@Override
public synchronized void init() {
    if (httpClient != null) {
        return;
    }
    if (logger.isDebugEnabled()) {
        logger.debug("Initializing " + HcHttpClient.class.getName());
    }
    super.init();
    // robots.txt parser
    final Boolean robotsTxtEnabled = getInitParameter(ROBOTS_TXT_ENABLED_PROPERTY, Boolean.TRUE, Boolean.class);
    if (robotsTxtHelper != null) {
        robotsTxtHelper.setEnabled(robotsTxtEnabled.booleanValue());
    }
    // httpclient
    final org.apache.http.client.config.RequestConfig.Builder requestConfigBuilder = RequestConfig.custom();
    final HttpClientBuilder httpClientBuilder = HttpClientBuilder.create();
    final Integer connectionTimeoutParam = getInitParameter(CONNECTION_TIMEOUT_PROPERTY, connectionTimeout, Integer.class);
    if (connectionTimeoutParam != null) {
        requestConfigBuilder.setConnectTimeout(connectionTimeoutParam);
    }
    final Integer soTimeoutParam = getInitParameter(SO_TIMEOUT_PROPERTY, soTimeout, Integer.class);
    if (soTimeoutParam != null) {
        requestConfigBuilder.setSocketTimeout(soTimeoutParam);
    }
    // AuthSchemeFactory
    final RegistryBuilder<AuthSchemeProvider> authSchemeProviderBuilder = RegistryBuilder.create();
    @SuppressWarnings("unchecked") final Map<String, AuthSchemeProvider> factoryMap = getInitParameter(AUTH_SCHEME_PROVIDERS_PROPERTY, authSchemeProviderMap, Map.class);
    if (factoryMap != null) {
        for (final Map.Entry<String, AuthSchemeProvider> entry : factoryMap.entrySet()) {
            authSchemeProviderBuilder.register(entry.getKey(), entry.getValue());
        }
    }
    // user agent
    userAgent = getInitParameter(USER_AGENT_PROPERTY, userAgent, String.class);
    if (StringUtil.isNotBlank(userAgent)) {
        httpClientBuilder.setUserAgent(userAgent);
    }
    final HttpRoutePlanner planner = buildRoutePlanner();
    if (planner != null) {
        httpClientBuilder.setRoutePlanner(planner);
    }
    // Authentication
    final Authentication[] siteCredentialList = getInitParameter(BASIC_AUTHENTICATIONS_PROPERTY, new Authentication[0], Authentication[].class);
    final List<Pair<FormScheme, Credentials>> formSchemeList = new ArrayList<>();
    for (final Authentication authentication : siteCredentialList) {
        final AuthScheme authScheme = authentication.getAuthScheme();
        if (authScheme instanceof FormScheme) {
            formSchemeList.add(new Pair<>((FormScheme) authScheme, authentication.getCredentials()));
        } else {
            final AuthScope authScope = authentication.getAuthScope();
            credentialsProvider.setCredentials(authScope, authentication.getCredentials());
            if (authScope.getHost() != null && authScheme != null) {
                final HttpHost targetHost = new HttpHost(authScope.getHost(), authScope.getPort());
                authCache.put(targetHost, authScheme);
            }
        }
    }
    httpClientContext.setAuthCache(authCache);
    httpClientContext.setCredentialsProvider(credentialsProvider);
    // Request Header
    final RequestHeader[] requestHeaders = getInitParameter(REQUERT_HEADERS_PROPERTY, new RequestHeader[0], RequestHeader[].class);
    for (final RequestHeader requestHeader : requestHeaders) {
        if (requestHeader.isValid()) {
            requestHeaderList.add(new BasicHeader(requestHeader.getName(), requestHeader.getValue()));
        }
    }
    // do not redirect
    requestConfigBuilder.setRedirectsEnabled(getInitParameter(REDIRECTS_ENABLED, redirectsEnabled, Boolean.class));
    // cookie
    if (cookieSpec != null) {
        requestConfigBuilder.setCookieSpec(cookieSpec);
    }
    // cookie store
    httpClientBuilder.setDefaultCookieStore(cookieStore);
    if (cookieStore != null) {
        final Cookie[] cookies = getInitParameter(COOKIES_PROPERTY, new Cookie[0], Cookie[].class);
        for (final Cookie cookie : cookies) {
            cookieStore.addCookie(cookie);
        }
    }
    // cookie registry
    final Lookup<CookieSpecProvider> cookieSpecRegistry = buildCookieSpecRegistry();
    if (cookieSpecRegistry != null) {
        httpClientBuilder.setDefaultCookieSpecRegistry(cookieSpecRegistry);
    }
    // SSL
    final LayeredConnectionSocketFactory sslSocketFactory = buildSSLSocketFactory();
    if (sslSocketFactory != null) {
        httpClientBuilder.setSSLSocketFactory(sslSocketFactory);
    }
    connectionMonitorTask = TimeoutManager.getInstance().addTimeoutTarget(new HcConnectionMonitorTarget(clientConnectionManager, idleConnectionTimeout), connectionCheckInterval, true);
    final CloseableHttpClient closeableHttpClient = httpClientBuilder.setDnsResolver(dnsResolver).setConnectionManager(clientConnectionManager).setDefaultRequestConfig(requestConfigBuilder.build()).build();
    if (!httpClientPropertyMap.isEmpty()) {
        final BeanDesc beanDesc = BeanDescFactory.getBeanDesc(closeableHttpClient.getClass());
        for (final Map.Entry<String, Object> entry : httpClientPropertyMap.entrySet()) {
            final String propertyName = entry.getKey();
            if (beanDesc.hasPropertyDesc(propertyName)) {
                final PropertyDesc propertyDesc = beanDesc.getPropertyDesc(propertyName);
                propertyDesc.setValue(closeableHttpClient, entry.getValue());
            } else {
                logger.warn("DefaultHttpClient does not have " + propertyName + ".");
            }
        }
    }
    formSchemeList.forEach(p -> {
        final FormScheme scheme = p.getFirst();
        final Credentials credentials = p.getSecond();
        scheme.authenticate(credentials, (request, consumer) -> {
            // request header
            for (final Header header : requestHeaderList) {
                request.addHeader(header);
            }
            HttpEntity httpEntity = null;
            try {
                final HttpResponse response = closeableHttpClient.execute(request, new BasicHttpContext(httpClientContext));
                httpEntity = response.getEntity();
                consumer.accept(response, httpEntity);
            } catch (final Exception e) {
                request.abort();
                logger.warn("Failed to authenticate on " + scheme, e);
            } finally {
                EntityUtils.consumeQuietly(httpEntity);
            }
        });
    });
    httpClient = closeableHttpClient;
}

Example 28 with AuthScheme

use of org.apache.http.auth.AuthScheme in project fess-crawler by codelibs.

the class ApiExtractor method init.

@PostConstruct
public void init() {
    if (logger.isDebugEnabled()) {
        logger.debug("Initializing " + ApiExtractor.class.getName());
    }
    // httpclient
    final org.apache.http.client.config.RequestConfig.Builder requestConfigBuilder = RequestConfig.custom();
    final HttpClientBuilder httpClientBuilder = HttpClientBuilder.create();
    final Integer connectionTimeoutParam = connectionTimeout;
    if (connectionTimeoutParam != null) {
        requestConfigBuilder.setConnectTimeout(connectionTimeoutParam);
    }
    final Integer soTimeoutParam = soTimeout;
    if (soTimeoutParam != null) {
        requestConfigBuilder.setSocketTimeout(soTimeoutParam);
    }
    // AuthSchemeFactory
    final RegistryBuilder<AuthSchemeProvider> authSchemeProviderBuilder = RegistryBuilder.create();
    // @SuppressWarnings("unchecked")
    final Map<String, AuthSchemeProvider> factoryMap = authSchemeProviderMap;
    if (factoryMap != null) {
        for (final Map.Entry<String, AuthSchemeProvider> entry : factoryMap.entrySet()) {
            authSchemeProviderBuilder.register(entry.getKey(), entry.getValue());
        }
    }
    // user agent
    if (StringUtil.isNotBlank(userAgent)) {
        httpClientBuilder.setUserAgent(userAgent);
    }
    // Authentication
    final Authentication[] siteCredentialList = new Authentication[0];
    for (final Authentication authentication : siteCredentialList) {
        final AuthScope authScope = authentication.getAuthScope();
        credentialsProvider.setCredentials(authScope, authentication.getCredentials());
        final AuthScheme authScheme = authentication.getAuthScheme();
        if (authScope.getHost() != null && authScheme != null) {
            final HttpHost targetHost = new HttpHost(authScope.getHost(), authScope.getPort());
            authCache.put(targetHost, authScheme);
        }
    }
    httpClientContext.setAuthCache(authCache);
    httpClientContext.setCredentialsProvider(credentialsProvider);
    // Request Header
    final RequestHeader[] requestHeaders = { new RequestHeader("enctype", "multipart/form-data") };
    for (final RequestHeader requestHeader : requestHeaders) {
        if (requestHeader.isValid()) {
            requestHeaderList.add(new BasicHeader(requestHeader.getName(), requestHeader.getValue()));
        }
    }
    final CloseableHttpClient closeableHttpClient = httpClientBuilder.setDefaultRequestConfig(requestConfigBuilder.build()).build();
    if (!httpClientPropertyMap.isEmpty()) {
        final BeanDesc beanDesc = BeanDescFactory.getBeanDesc(closeableHttpClient.getClass());
        for (final Map.Entry<String, Object> entry : httpClientPropertyMap.entrySet()) {
            final String propertyName = entry.getKey();
            if (beanDesc.hasPropertyDesc(propertyName)) {
                final PropertyDesc propertyDesc = beanDesc.getPropertyDesc(propertyName);
                propertyDesc.setValue(closeableHttpClient, entry.getValue());
            } else {
                logger.warn("DefaultHttpClient does not have " + propertyName + ".");
            }
        }
    }
    httpClient = closeableHttpClient;
}

Example 29 with AuthScheme

use of org.apache.http.auth.AuthScheme in project fess by codelibs.

the class DataConfig method getAuthScheme.

private AuthScheme getAuthScheme(final Map<String, String> paramMap, final String webAuthName, final String scheme) {
    AuthScheme authScheme = null;
    if (Constants.BASIC.equals(scheme)) {
        authScheme = new BasicScheme();
    } else if (Constants.DIGEST.equals(scheme)) {
        authScheme = new DigestScheme();
    } else if (Constants.NTLM.equals(scheme)) {
        final Properties props = new Properties();
        paramMap.entrySet().stream().filter(e -> e.getKey().startsWith("jcifs.")).forEach(e -> {
            props.setProperty(e.getKey(), e.getValue());
        });
        authScheme = new NTLMScheme(new JcifsEngine(props));
    } else if (Constants.FORM.equals(scheme)) {
        final String prefix = CRAWLER_WEB_AUTH + "." + webAuthName + ".";
        final Map<String, String> parameterMap = paramMap.entrySet().stream().filter(e -> e.getKey().startsWith(prefix)).collect(Collectors.toMap(e -> e.getKey().substring(prefix.length()), Entry::getValue));
        authScheme = new FormScheme(parameterMap);
    }
    return authScheme;
}

Example 30 with AuthScheme

use of org.apache.http.auth.AuthScheme in project fess by codelibs.

the class DataConfig method initializeClientFactory.

@Override
public CrawlerClientFactory initializeClientFactory(final Supplier<CrawlerClientFactory> creator) {
    if (crawlerClientFactory != null) {
        return crawlerClientFactory;
    }
    final CrawlerClientFactory factory = creator.get();
    final Map<String, String> paramMap = getHandlerParameterMap();
    final Map<String, Object> factoryParamMap = new HashMap<>();
    factory.setInitParameterMap(factoryParamMap);
    // parameters
    for (final Map.Entry<String, String> entry : paramMap.entrySet()) {
        final String key = entry.getKey();
        if (key.startsWith(CRAWLER_PARAM_PREFIX)) {
            factoryParamMap.put(key.substring(CRAWLER_PARAM_PREFIX.length()), entry.getValue());
        }
    }
    // user agent
    final String userAgent = paramMap.get(CRAWLER_USERAGENT);
    if (StringUtil.isNotBlank(userAgent)) {
        factoryParamMap.put(HcHttpClient.USER_AGENT_PROPERTY, userAgent);
    }
    // web auth
    final String webAuthStr = paramMap.get(CRAWLER_WEB_AUTH);
    if (StringUtil.isNotBlank(webAuthStr)) {
        final String[] webAuthNames = webAuthStr.split(",");
        final List<Authentication> basicAuthList = new ArrayList<>();
        for (final String webAuthName : webAuthNames) {
            final String scheme = paramMap.get(CRAWLER_WEB_AUTH + "." + webAuthName + ".scheme");
            final AuthScheme authScheme = getAuthScheme(paramMap, webAuthName, scheme);
            final AuthScope authScope = getAuthScope(webAuthName, scheme, paramMap);
            final Credentials credentials = getCredentials(webAuthName, scheme, paramMap);
            basicAuthList.add(new AuthenticationImpl(authScope, credentials, authScheme));
        }
        factoryParamMap.put(HcHttpClient.BASIC_AUTHENTICATIONS_PROPERTY, basicAuthList.toArray(new Authentication[basicAuthList.size()]));
    }
    // request header
    final List<org.codelibs.fess.crawler.client.http.RequestHeader> rhList = new ArrayList<>();
    int count = 1;
    String headerName = paramMap.get(CRAWLER_WEB_HEADER_PREFIX + count + ".name");
    while (StringUtil.isNotBlank(headerName)) {
        final String headerValue = paramMap.get(CRAWLER_WEB_HEADER_PREFIX + count + ".value");
        rhList.add(new org.codelibs.fess.crawler.client.http.RequestHeader(headerName, headerValue));
        count++;
        headerName = paramMap.get(CRAWLER_WEB_HEADER_PREFIX + count + ".name");
    }
    if (!rhList.isEmpty()) {
        factoryParamMap.put(HcHttpClient.REQUERT_HEADERS_PROPERTY, rhList.toArray(new org.codelibs.fess.crawler.client.http.RequestHeader[rhList.size()]));
    }
    // proxy credentials
    final String proxyHost = paramMap.get(CRAWLER_WEB_PREFIX + "proxyHost");
    final String proxyPort = paramMap.get(CRAWLER_WEB_PREFIX + "proxyPort");
    if (StringUtil.isNotBlank(proxyHost) && StringUtil.isNotBlank(proxyPort)) {
        factoryParamMap.put(HcHttpClient.PROXY_HOST_PROPERTY, proxyHost);
        factoryParamMap.put(HcHttpClient.PROXY_PORT_PROPERTY, proxyPort);
        final String proxyUsername = paramMap.get(CRAWLER_WEB_PREFIX + "proxyUsername");
        final String proxyPassword = paramMap.get(CRAWLER_WEB_PREFIX + "proxyPassword");
        if (proxyUsername != null && proxyPassword != null) {
            factoryParamMap.put(HcHttpClient.PROXY_CREDENTIALS_PROPERTY, new UsernamePasswordCredentials(proxyUsername, proxyPassword));
        }
    } else {
        initializeDefaultHttpProxy(factoryParamMap);
    }
    // file auth
    final String fileAuthStr = paramMap.get(CRAWLER_FILE_AUTH);
    if (StringUtil.isNotBlank(fileAuthStr)) {
        final String[] fileAuthNames = fileAuthStr.split(",");
        final List<SmbAuthentication> smbAuthList = new ArrayList<>();
        final List<org.codelibs.fess.crawler.client.smb1.SmbAuthentication> smb1AuthList = new ArrayList<>();
        final List<FtpAuthentication> ftpAuthList = new ArrayList<>();
        for (final String fileAuthName : fileAuthNames) {
            final String scheme = paramMap.get(CRAWLER_FILE_AUTH + "." + fileAuthName + ".scheme");
            if (Constants.SAMBA.equals(scheme)) {
                final String domain = paramMap.get(CRAWLER_FILE_AUTH + "." + fileAuthName + ".domain");
                final String hostname = paramMap.get(CRAWLER_FILE_AUTH + "." + fileAuthName + ".host");
                final String port = paramMap.get(CRAWLER_FILE_AUTH + "." + fileAuthName + ".port");
                final String username = paramMap.get(CRAWLER_FILE_AUTH + "." + fileAuthName + ".username");
                final String password = paramMap.get(CRAWLER_FILE_AUTH + "." + fileAuthName + ".password");
                if (StringUtil.isEmpty(username)) {
                    logger.warn("username is empty. fileAuth:{}", fileAuthName);
                    continue;
                }
                final SmbAuthentication smbAuth = new SmbAuthentication();
                smbAuth.setDomain(domain == null ? StringUtil.EMPTY : domain);
                smbAuth.setServer(hostname);
                if (StringUtil.isNotBlank(port)) {
                    try {
                        smbAuth.setPort(Integer.parseInt(port));
                    } catch (final NumberFormatException e) {
                        logger.warn("Failed to parse {}", port, e);
                    }
                }
                smbAuth.setUsername(username);
                smbAuth.setPassword(password == null ? StringUtil.EMPTY : password);
                smbAuthList.add(smbAuth);
                final org.codelibs.fess.crawler.client.smb1.SmbAuthentication smb1Auth = new org.codelibs.fess.crawler.client.smb1.SmbAuthentication();
                smb1Auth.setDomain(domain == null ? StringUtil.EMPTY : domain);
                smb1Auth.setServer(hostname);
                if (StringUtil.isNotBlank(port)) {
                    try {
                        smb1Auth.setPort(Integer.parseInt(port));
                    } catch (final NumberFormatException e) {
                        logger.warn("Failed to parse {}", port, e);
                    }
                }
                smb1Auth.setUsername(username);
                smb1Auth.setPassword(password == null ? StringUtil.EMPTY : password);
                smb1AuthList.add(smb1Auth);
            } else if (Constants.FTP.equals(scheme)) {
                final String hostname = paramMap.get(CRAWLER_FILE_AUTH + "." + fileAuthName + ".host");
                final String port = paramMap.get(CRAWLER_FILE_AUTH + "." + fileAuthName + ".port");
                final String username = paramMap.get(CRAWLER_FILE_AUTH + "." + fileAuthName + ".username");
                final String password = paramMap.get(CRAWLER_FILE_AUTH + "." + fileAuthName + ".password");
                if (StringUtil.isEmpty(username)) {
                    logger.warn("username is empty. fileAuth:{}", fileAuthName);
                    continue;
                }
                final FtpAuthentication ftpAuth = new FtpAuthentication();
                ftpAuth.setServer(hostname);
                if (StringUtil.isNotBlank(port)) {
                    try {
                        ftpAuth.setPort(Integer.parseInt(port));
                    } catch (final NumberFormatException e) {
                        logger.warn("Failed to parse {}", port, e);
                    }
                }
                ftpAuth.setUsername(username);
                ftpAuth.setPassword(password == null ? StringUtil.EMPTY : password);
                ftpAuthList.add(ftpAuth);
            }
        }
        if (!smbAuthList.isEmpty()) {
            factoryParamMap.put(SmbClient.SMB_AUTHENTICATIONS_PROPERTY, smbAuthList.toArray(new SmbAuthentication[smbAuthList.size()]));
        }
        if (!smb1AuthList.isEmpty()) {
            factoryParamMap.put(org.codelibs.fess.crawler.client.smb1.SmbClient.SMB_AUTHENTICATIONS_PROPERTY, smb1AuthList.toArray(new org.codelibs.fess.crawler.client.smb1.SmbAuthentication[smb1AuthList.size()]));
        }
        if (!ftpAuthList.isEmpty()) {
            factoryParamMap.put(FtpClient.FTP_AUTHENTICATIONS_PROPERTY, ftpAuthList.toArray(new FtpAuthentication[ftpAuthList.size()]));
        }
    }
    crawlerClientFactory = factory;
    return factory;
}