Examples with FormScheme org.codelibs.fess.crawler.client.http.form.FormScheme used on opensource projects

Example 1 with FormScheme

use of org.codelibs.fess.crawler.client.http.form.FormScheme in project fess by codelibs.

the class WebAuthentication method getAuthScheme.

private AuthScheme getAuthScheme() {
    final String scheme = getProtocolScheme();
    if (Constants.BASIC.equals(scheme)) {
        return new BasicScheme();
    }
    if (Constants.DIGEST.equals(scheme)) {
        return new DigestScheme();
    }
    if (Constants.NTLM.equals(scheme)) {
        final Properties props = new Properties();
        getWebConfig().getConfigParameterMap(ConfigName.CONFIG).entrySet().stream().filter(e -> e.getKey().startsWith(Config.JCIFS_PREFIX)).forEach(e -> {
            props.setProperty(e.getKey(), e.getValue());
        });
        return new NTLMScheme(new JcifsEngine(props));
    }
    if (Constants.FORM.equals(scheme)) {
        final Map<String, String> parameterMap = ParameterUtil.parse(getParameters());
        return new FormScheme(parameterMap);
    }
    return null;
}

Example 2 with FormScheme

use of org.codelibs.fess.crawler.client.http.form.FormScheme in project fess-crawler by codelibs.

the class HcHttpClient method init.

@Override
public synchronized void init() {
    if (httpClient != null) {
        return;
    }
    if (logger.isDebugEnabled()) {
        logger.debug("Initializing " + HcHttpClient.class.getName());
    }
    super.init();
    // robots.txt parser
    final Boolean robotsTxtEnabled = getInitParameter(ROBOTS_TXT_ENABLED_PROPERTY, Boolean.TRUE, Boolean.class);
    if (robotsTxtHelper != null) {
        robotsTxtHelper.setEnabled(robotsTxtEnabled.booleanValue());
    }
    // httpclient
    final org.apache.http.client.config.RequestConfig.Builder requestConfigBuilder = RequestConfig.custom();
    final HttpClientBuilder httpClientBuilder = HttpClientBuilder.create();
    final Integer connectionTimeoutParam = getInitParameter(CONNECTION_TIMEOUT_PROPERTY, connectionTimeout, Integer.class);
    if (connectionTimeoutParam != null) {
        requestConfigBuilder.setConnectTimeout(connectionTimeoutParam);
    }
    final Integer soTimeoutParam = getInitParameter(SO_TIMEOUT_PROPERTY, soTimeout, Integer.class);
    if (soTimeoutParam != null) {
        requestConfigBuilder.setSocketTimeout(soTimeoutParam);
    }
    // AuthSchemeFactory
    final RegistryBuilder<AuthSchemeProvider> authSchemeProviderBuilder = RegistryBuilder.create();
    @SuppressWarnings("unchecked") final Map<String, AuthSchemeProvider> factoryMap = getInitParameter(AUTH_SCHEME_PROVIDERS_PROPERTY, authSchemeProviderMap, Map.class);
    if (factoryMap != null) {
        for (final Map.Entry<String, AuthSchemeProvider> entry : factoryMap.entrySet()) {
            authSchemeProviderBuilder.register(entry.getKey(), entry.getValue());
        }
    }
    // user agent
    userAgent = getInitParameter(USER_AGENT_PROPERTY, userAgent, String.class);
    if (StringUtil.isNotBlank(userAgent)) {
        httpClientBuilder.setUserAgent(userAgent);
    }
    final HttpRoutePlanner planner = buildRoutePlanner();
    if (planner != null) {
        httpClientBuilder.setRoutePlanner(planner);
    }
    // Authentication
    final Authentication[] siteCredentialList = getInitParameter(BASIC_AUTHENTICATIONS_PROPERTY, new Authentication[0], Authentication[].class);
    final List<Pair<FormScheme, Credentials>> formSchemeList = new ArrayList<>();
    for (final Authentication authentication : siteCredentialList) {
        final AuthScheme authScheme = authentication.getAuthScheme();
        if (authScheme instanceof FormScheme) {
            formSchemeList.add(new Pair<>((FormScheme) authScheme, authentication.getCredentials()));
        } else {
            final AuthScope authScope = authentication.getAuthScope();
            credentialsProvider.setCredentials(authScope, authentication.getCredentials());
            if (authScope.getHost() != null && authScheme != null) {
                final HttpHost targetHost = new HttpHost(authScope.getHost(), authScope.getPort());
                authCache.put(targetHost, authScheme);
            }
        }
    }
    httpClientContext.setAuthCache(authCache);
    httpClientContext.setCredentialsProvider(credentialsProvider);
    // Request Header
    final RequestHeader[] requestHeaders = getInitParameter(REQUERT_HEADERS_PROPERTY, new RequestHeader[0], RequestHeader[].class);
    for (final RequestHeader requestHeader : requestHeaders) {
        if (requestHeader.isValid()) {
            requestHeaderList.add(new BasicHeader(requestHeader.getName(), requestHeader.getValue()));
        }
    }
    // do not redirect
    requestConfigBuilder.setRedirectsEnabled(getInitParameter(REDIRECTS_ENABLED, redirectsEnabled, Boolean.class));
    // cookie
    if (cookieSpec != null) {
        requestConfigBuilder.setCookieSpec(cookieSpec);
    }
    // cookie store
    httpClientBuilder.setDefaultCookieStore(cookieStore);
    if (cookieStore != null) {
        final Cookie[] cookies = getInitParameter(COOKIES_PROPERTY, new Cookie[0], Cookie[].class);
        for (final Cookie cookie : cookies) {
            cookieStore.addCookie(cookie);
        }
    }
    // cookie registry
    final Lookup<CookieSpecProvider> cookieSpecRegistry = buildCookieSpecRegistry();
    if (cookieSpecRegistry != null) {
        httpClientBuilder.setDefaultCookieSpecRegistry(cookieSpecRegistry);
    }
    // SSL
    final LayeredConnectionSocketFactory sslSocketFactory = buildSSLSocketFactory();
    if (sslSocketFactory != null) {
        httpClientBuilder.setSSLSocketFactory(sslSocketFactory);
    }
    connectionMonitorTask = TimeoutManager.getInstance().addTimeoutTarget(new HcConnectionMonitorTarget(clientConnectionManager, idleConnectionTimeout), connectionCheckInterval, true);
    final CloseableHttpClient closeableHttpClient = httpClientBuilder.setDnsResolver(dnsResolver).setConnectionManager(clientConnectionManager).setDefaultRequestConfig(requestConfigBuilder.build()).build();
    if (!httpClientPropertyMap.isEmpty()) {
        final BeanDesc beanDesc = BeanDescFactory.getBeanDesc(closeableHttpClient.getClass());
        for (final Map.Entry<String, Object> entry : httpClientPropertyMap.entrySet()) {
            final String propertyName = entry.getKey();
            if (beanDesc.hasPropertyDesc(propertyName)) {
                final PropertyDesc propertyDesc = beanDesc.getPropertyDesc(propertyName);
                propertyDesc.setValue(closeableHttpClient, entry.getValue());
            } else {
                logger.warn("DefaultHttpClient does not have " + propertyName + ".");
            }
        }
    }
    formSchemeList.forEach(p -> {
        final FormScheme scheme = p.getFirst();
        final Credentials credentials = p.getSecond();
        scheme.authenticate(credentials, (request, consumer) -> {
            // request header
            for (final Header header : requestHeaderList) {
                request.addHeader(header);
            }
            HttpEntity httpEntity = null;
            try {
                final HttpResponse response = closeableHttpClient.execute(request, new BasicHttpContext(httpClientContext));
                httpEntity = response.getEntity();
                consumer.accept(response, httpEntity);
            } catch (final Exception e) {
                request.abort();
                logger.warn("Failed to authenticate on " + scheme, e);
            } finally {
                EntityUtils.consumeQuietly(httpEntity);
            }
        });
    });
    httpClient = closeableHttpClient;
}

Example 3 with FormScheme

use of org.codelibs.fess.crawler.client.http.form.FormScheme in project fess by codelibs.

the class DataConfig method getAuthScheme.

private AuthScheme getAuthScheme(final Map<String, String> paramMap, final String webAuthName, final String scheme) {
    AuthScheme authScheme = null;
    if (Constants.BASIC.equals(scheme)) {
        authScheme = new BasicScheme();
    } else if (Constants.DIGEST.equals(scheme)) {
        authScheme = new DigestScheme();
    } else if (Constants.NTLM.equals(scheme)) {
        final Properties props = new Properties();
        paramMap.entrySet().stream().filter(e -> e.getKey().startsWith("jcifs.")).forEach(e -> {
            props.setProperty(e.getKey(), e.getValue());
        });
        authScheme = new NTLMScheme(new JcifsEngine(props));
    } else if (Constants.FORM.equals(scheme)) {
        final String prefix = CRAWLER_WEB_AUTH + "." + webAuthName + ".";
        final Map<String, String> parameterMap = paramMap.entrySet().stream().filter(e -> e.getKey().startsWith(prefix)).collect(Collectors.toMap(e -> e.getKey().substring(prefix.length()), Entry::getValue));
        authScheme = new FormScheme(parameterMap);
    }
    return authScheme;
}