use of org.codelibs.fess.crawler.client.ftp.FtpAuthentication in project fess by codelibs.
the class FileConfig method initializeClientFactory.
@Override
public Map<String, Object> initializeClientFactory(final CrawlerClientFactory clientFactory) {
final FileAuthenticationService fileAuthenticationService = ComponentUtil.getComponent(FileAuthenticationService.class);
// Parameters
final Map<String, Object> paramMap = new HashMap<>();
clientFactory.setInitParameterMap(paramMap);
final Map<String, String> clientConfigMap = getConfigParameterMap(ConfigName.CLIENT);
if (clientConfigMap != null) {
paramMap.putAll(clientConfigMap);
}
// auth params
final List<FileAuthentication> fileAuthList = fileAuthenticationService.getFileAuthenticationList(getId());
final List<SmbAuthentication> smbAuthList = new ArrayList<>();
final List<FtpAuthentication> ftpAuthList = new ArrayList<>();
for (final FileAuthentication fileAuth : fileAuthList) {
if (Constants.SAMBA.equals(fileAuth.getProtocolScheme())) {
final SmbAuthentication smbAuth = new SmbAuthentication();
final Map<String, String> map = ParameterUtil.parse(fileAuth.getParameters());
final String domain = map.get("domain");
smbAuth.setDomain(domain == null ? StringUtil.EMPTY : domain);
smbAuth.setServer(fileAuth.getHostname());
smbAuth.setPort(fileAuth.getPort() == null ? -1 : fileAuth.getPort().intValue());
smbAuth.setUsername(fileAuth.getUsername());
smbAuth.setPassword(fileAuth.getPassword());
smbAuthList.add(smbAuth);
} else if (Constants.FTP.equals(fileAuth.getProtocolScheme())) {
final FtpAuthentication ftpAuth = new FtpAuthentication();
ftpAuth.setServer(fileAuth.getHostname());
ftpAuth.setPort(fileAuth.getPort());
ftpAuth.setUsername(fileAuth.getUsername());
ftpAuth.setPassword(fileAuth.getPassword());
ftpAuthList.add(ftpAuth);
}
}
paramMap.put(SmbClient.SMB_AUTHENTICATIONS_PROPERTY, smbAuthList.toArray(new SmbAuthentication[smbAuthList.size()]));
paramMap.put(FtpClient.FTP_AUTHENTICATIONS_PROPERTY, ftpAuthList.toArray(new FtpAuthentication[ftpAuthList.size()]));
return paramMap;
}
use of org.codelibs.fess.crawler.client.ftp.FtpAuthentication in project fess by codelibs.
the class DataConfig method initializeClientFactory.
@Override
public Map<String, Object> initializeClientFactory(final CrawlerClientFactory crawlerClientFactory) {
final Map<String, String> paramMap = getHandlerParameterMap();
final Map<String, Object> factoryParamMap = new HashMap<>();
crawlerClientFactory.setInitParameterMap(factoryParamMap);
// parameters
for (final Map.Entry<String, String> entry : paramMap.entrySet()) {
final String key = entry.getKey();
if (key.startsWith(CRAWLER_PARAM_PREFIX)) {
factoryParamMap.put(key.substring(CRAWLER_PARAM_PREFIX.length()), entry.getValue());
}
}
// user agent
final String userAgent = paramMap.get(CRAWLER_USERAGENT);
if (StringUtil.isNotBlank(userAgent)) {
factoryParamMap.put(HcHttpClient.USER_AGENT_PROPERTY, userAgent);
}
// web auth
final String webAuthStr = paramMap.get(CRAWLER_WEB_AUTH);
if (StringUtil.isNotBlank(webAuthStr)) {
final String[] webAuthNames = webAuthStr.split(",");
final List<Authentication> basicAuthList = new ArrayList<>();
for (final String webAuthName : webAuthNames) {
final String scheme = paramMap.get(CRAWLER_WEB_AUTH + "." + webAuthName + ".scheme");
final String hostname = paramMap.get(CRAWLER_WEB_AUTH + "." + webAuthName + ".host");
final String port = paramMap.get(CRAWLER_WEB_AUTH + "." + webAuthName + ".port");
final String realm = paramMap.get(CRAWLER_WEB_AUTH + "." + webAuthName + ".realm");
final String username = paramMap.get(CRAWLER_WEB_AUTH + "." + webAuthName + ".username");
final String password = paramMap.get(CRAWLER_WEB_AUTH + "." + webAuthName + ".password");
if (StringUtil.isEmpty(username)) {
logger.warn("username is empty. webAuth:" + webAuthName);
continue;
}
AuthScheme authScheme = null;
if (Constants.BASIC.equals(scheme)) {
authScheme = new BasicScheme();
} else if (Constants.DIGEST.equals(scheme)) {
authScheme = new DigestScheme();
} else if (Constants.NTLM.equals(scheme)) {
authScheme = new NTLMScheme(new JcifsEngine());
}
// TODO FORM
AuthScope authScope;
if (StringUtil.isBlank(hostname)) {
authScope = AuthScope.ANY;
} else {
int p = AuthScope.ANY_PORT;
if (StringUtil.isNotBlank(port)) {
try {
p = Integer.parseInt(port);
} catch (final NumberFormatException e) {
logger.warn("Failed to parse " + port, e);
}
}
String r = realm;
if (StringUtil.isBlank(realm)) {
r = AuthScope.ANY_REALM;
}
String s = scheme;
if (StringUtil.isBlank(scheme) || Constants.NTLM.equals(scheme)) {
s = AuthScope.ANY_SCHEME;
}
authScope = new AuthScope(hostname, p, r, s);
}
Credentials credentials;
if (Constants.NTLM.equals(scheme)) {
final String workstation = paramMap.get(CRAWLER_WEB_AUTH + "." + webAuthName + ".workstation");
final String domain = paramMap.get(CRAWLER_WEB_AUTH + "." + webAuthName + ".domain");
credentials = new NTCredentials(username, password == null ? StringUtil.EMPTY : password, workstation == null ? StringUtil.EMPTY : workstation, domain == null ? StringUtil.EMPTY : domain);
} else {
credentials = new UsernamePasswordCredentials(username, password == null ? StringUtil.EMPTY : password);
}
basicAuthList.add(new AuthenticationImpl(authScope, credentials, authScheme));
}
factoryParamMap.put(HcHttpClient.BASIC_AUTHENTICATIONS_PROPERTY, basicAuthList.toArray(new Authentication[basicAuthList.size()]));
}
// request header
final List<org.codelibs.fess.crawler.client.http.RequestHeader> rhList = new ArrayList<>();
int count = 1;
String headerName = paramMap.get(CRAWLER_WEB_HEADER_PREFIX + count + ".name");
while (StringUtil.isNotBlank(headerName)) {
final String headerValue = paramMap.get(CRAWLER_WEB_HEADER_PREFIX + count + ".value");
rhList.add(new org.codelibs.fess.crawler.client.http.RequestHeader(headerName, headerValue));
count++;
headerName = paramMap.get(CRAWLER_WEB_HEADER_PREFIX + count + ".name");
}
if (!rhList.isEmpty()) {
factoryParamMap.put(HcHttpClient.REQUERT_HEADERS_PROPERTY, rhList.toArray(new org.codelibs.fess.crawler.client.http.RequestHeader[rhList.size()]));
}
// file auth
final String fileAuthStr = paramMap.get(CRAWLER_FILE_AUTH);
if (StringUtil.isNotBlank(fileAuthStr)) {
final String[] fileAuthNames = fileAuthStr.split(",");
final List<SmbAuthentication> smbAuthList = new ArrayList<>();
final List<FtpAuthentication> ftpAuthList = new ArrayList<>();
for (final String fileAuthName : fileAuthNames) {
final String scheme = paramMap.get(CRAWLER_FILE_AUTH + "." + fileAuthName + ".scheme");
if (Constants.SAMBA.equals(scheme)) {
final String domain = paramMap.get(CRAWLER_FILE_AUTH + "." + fileAuthName + ".domain");
final String hostname = paramMap.get(CRAWLER_FILE_AUTH + "." + fileAuthName + ".host");
final String port = paramMap.get(CRAWLER_FILE_AUTH + "." + fileAuthName + ".port");
final String username = paramMap.get(CRAWLER_FILE_AUTH + "." + fileAuthName + ".username");
final String password = paramMap.get(CRAWLER_FILE_AUTH + "." + fileAuthName + ".password");
if (StringUtil.isEmpty(username)) {
logger.warn("username is empty. fileAuth:" + fileAuthName);
continue;
}
final SmbAuthentication smbAuth = new SmbAuthentication();
smbAuth.setDomain(domain == null ? StringUtil.EMPTY : domain);
smbAuth.setServer(hostname);
if (StringUtil.isNotBlank(port)) {
try {
smbAuth.setPort(Integer.parseInt(port));
} catch (final NumberFormatException e) {
logger.warn("Failed to parse " + port, e);
}
}
smbAuth.setUsername(username);
smbAuth.setPassword(password == null ? StringUtil.EMPTY : password);
smbAuthList.add(smbAuth);
} else if (Constants.FTP.equals(scheme)) {
final String hostname = paramMap.get(CRAWLER_FILE_AUTH + "." + fileAuthName + ".host");
final String port = paramMap.get(CRAWLER_FILE_AUTH + "." + fileAuthName + ".port");
final String username = paramMap.get(CRAWLER_FILE_AUTH + "." + fileAuthName + ".username");
final String password = paramMap.get(CRAWLER_FILE_AUTH + "." + fileAuthName + ".password");
if (StringUtil.isEmpty(username)) {
logger.warn("username is empty. fileAuth:" + fileAuthName);
continue;
}
final FtpAuthentication ftpAuth = new FtpAuthentication();
ftpAuth.setServer(hostname);
if (StringUtil.isNotBlank(port)) {
try {
ftpAuth.setPort(Integer.parseInt(port));
} catch (final NumberFormatException e) {
logger.warn("Failed to parse " + port, e);
}
}
ftpAuth.setUsername(username);
ftpAuth.setPassword(password == null ? StringUtil.EMPTY : password);
ftpAuthList.add(ftpAuth);
}
}
if (!smbAuthList.isEmpty()) {
factoryParamMap.put(SmbClient.SMB_AUTHENTICATIONS_PROPERTY, smbAuthList.toArray(new SmbAuthentication[smbAuthList.size()]));
}
if (!ftpAuthList.isEmpty()) {
factoryParamMap.put(FtpClient.FTP_AUTHENTICATIONS_PROPERTY, ftpAuthList.toArray(new FtpAuthentication[ftpAuthList.size()]));
}
}
return factoryParamMap;
}
use of org.codelibs.fess.crawler.client.ftp.FtpAuthentication in project fess by codelibs.
the class DataConfig method initializeClientFactory.
@Override
public CrawlerClientFactory initializeClientFactory(final Supplier<CrawlerClientFactory> creator) {
if (crawlerClientFactory != null) {
return crawlerClientFactory;
}
final CrawlerClientFactory factory = creator.get();
final Map<String, String> paramMap = getHandlerParameterMap();
final Map<String, Object> factoryParamMap = new HashMap<>();
factory.setInitParameterMap(factoryParamMap);
// parameters
for (final Map.Entry<String, String> entry : paramMap.entrySet()) {
final String key = entry.getKey();
if (key.startsWith(CRAWLER_PARAM_PREFIX)) {
factoryParamMap.put(key.substring(CRAWLER_PARAM_PREFIX.length()), entry.getValue());
}
}
// user agent
final String userAgent = paramMap.get(CRAWLER_USERAGENT);
if (StringUtil.isNotBlank(userAgent)) {
factoryParamMap.put(HcHttpClient.USER_AGENT_PROPERTY, userAgent);
}
// web auth
final String webAuthStr = paramMap.get(CRAWLER_WEB_AUTH);
if (StringUtil.isNotBlank(webAuthStr)) {
final String[] webAuthNames = webAuthStr.split(",");
final List<Authentication> basicAuthList = new ArrayList<>();
for (final String webAuthName : webAuthNames) {
final String scheme = paramMap.get(CRAWLER_WEB_AUTH + "." + webAuthName + ".scheme");
final AuthScheme authScheme = getAuthScheme(paramMap, webAuthName, scheme);
final AuthScope authScope = getAuthScope(webAuthName, scheme, paramMap);
final Credentials credentials = getCredentials(webAuthName, scheme, paramMap);
basicAuthList.add(new AuthenticationImpl(authScope, credentials, authScheme));
}
factoryParamMap.put(HcHttpClient.BASIC_AUTHENTICATIONS_PROPERTY, basicAuthList.toArray(new Authentication[basicAuthList.size()]));
}
// request header
final List<org.codelibs.fess.crawler.client.http.RequestHeader> rhList = new ArrayList<>();
int count = 1;
String headerName = paramMap.get(CRAWLER_WEB_HEADER_PREFIX + count + ".name");
while (StringUtil.isNotBlank(headerName)) {
final String headerValue = paramMap.get(CRAWLER_WEB_HEADER_PREFIX + count + ".value");
rhList.add(new org.codelibs.fess.crawler.client.http.RequestHeader(headerName, headerValue));
count++;
headerName = paramMap.get(CRAWLER_WEB_HEADER_PREFIX + count + ".name");
}
if (!rhList.isEmpty()) {
factoryParamMap.put(HcHttpClient.REQUERT_HEADERS_PROPERTY, rhList.toArray(new org.codelibs.fess.crawler.client.http.RequestHeader[rhList.size()]));
}
// proxy credentials
final String proxyHost = paramMap.get(CRAWLER_WEB_PREFIX + "proxyHost");
final String proxyPort = paramMap.get(CRAWLER_WEB_PREFIX + "proxyPort");
if (StringUtil.isNotBlank(proxyHost) && StringUtil.isNotBlank(proxyPort)) {
factoryParamMap.put(HcHttpClient.PROXY_HOST_PROPERTY, proxyHost);
factoryParamMap.put(HcHttpClient.PROXY_PORT_PROPERTY, proxyPort);
final String proxyUsername = paramMap.get(CRAWLER_WEB_PREFIX + "proxyUsername");
final String proxyPassword = paramMap.get(CRAWLER_WEB_PREFIX + "proxyPassword");
if (proxyUsername != null && proxyPassword != null) {
factoryParamMap.put(HcHttpClient.PROXY_CREDENTIALS_PROPERTY, new UsernamePasswordCredentials(proxyUsername, proxyPassword));
}
} else {
initializeDefaultHttpProxy(factoryParamMap);
}
// file auth
final String fileAuthStr = paramMap.get(CRAWLER_FILE_AUTH);
if (StringUtil.isNotBlank(fileAuthStr)) {
final String[] fileAuthNames = fileAuthStr.split(",");
final List<SmbAuthentication> smbAuthList = new ArrayList<>();
final List<org.codelibs.fess.crawler.client.smb1.SmbAuthentication> smb1AuthList = new ArrayList<>();
final List<FtpAuthentication> ftpAuthList = new ArrayList<>();
for (final String fileAuthName : fileAuthNames) {
final String scheme = paramMap.get(CRAWLER_FILE_AUTH + "." + fileAuthName + ".scheme");
if (Constants.SAMBA.equals(scheme)) {
final String domain = paramMap.get(CRAWLER_FILE_AUTH + "." + fileAuthName + ".domain");
final String hostname = paramMap.get(CRAWLER_FILE_AUTH + "." + fileAuthName + ".host");
final String port = paramMap.get(CRAWLER_FILE_AUTH + "." + fileAuthName + ".port");
final String username = paramMap.get(CRAWLER_FILE_AUTH + "." + fileAuthName + ".username");
final String password = paramMap.get(CRAWLER_FILE_AUTH + "." + fileAuthName + ".password");
if (StringUtil.isEmpty(username)) {
logger.warn("username is empty. fileAuth:{}", fileAuthName);
continue;
}
final SmbAuthentication smbAuth = new SmbAuthentication();
smbAuth.setDomain(domain == null ? StringUtil.EMPTY : domain);
smbAuth.setServer(hostname);
if (StringUtil.isNotBlank(port)) {
try {
smbAuth.setPort(Integer.parseInt(port));
} catch (final NumberFormatException e) {
logger.warn("Failed to parse {}", port, e);
}
}
smbAuth.setUsername(username);
smbAuth.setPassword(password == null ? StringUtil.EMPTY : password);
smbAuthList.add(smbAuth);
final org.codelibs.fess.crawler.client.smb1.SmbAuthentication smb1Auth = new org.codelibs.fess.crawler.client.smb1.SmbAuthentication();
smb1Auth.setDomain(domain == null ? StringUtil.EMPTY : domain);
smb1Auth.setServer(hostname);
if (StringUtil.isNotBlank(port)) {
try {
smb1Auth.setPort(Integer.parseInt(port));
} catch (final NumberFormatException e) {
logger.warn("Failed to parse {}", port, e);
}
}
smb1Auth.setUsername(username);
smb1Auth.setPassword(password == null ? StringUtil.EMPTY : password);
smb1AuthList.add(smb1Auth);
} else if (Constants.FTP.equals(scheme)) {
final String hostname = paramMap.get(CRAWLER_FILE_AUTH + "." + fileAuthName + ".host");
final String port = paramMap.get(CRAWLER_FILE_AUTH + "." + fileAuthName + ".port");
final String username = paramMap.get(CRAWLER_FILE_AUTH + "." + fileAuthName + ".username");
final String password = paramMap.get(CRAWLER_FILE_AUTH + "." + fileAuthName + ".password");
if (StringUtil.isEmpty(username)) {
logger.warn("username is empty. fileAuth:{}", fileAuthName);
continue;
}
final FtpAuthentication ftpAuth = new FtpAuthentication();
ftpAuth.setServer(hostname);
if (StringUtil.isNotBlank(port)) {
try {
ftpAuth.setPort(Integer.parseInt(port));
} catch (final NumberFormatException e) {
logger.warn("Failed to parse {}", port, e);
}
}
ftpAuth.setUsername(username);
ftpAuth.setPassword(password == null ? StringUtil.EMPTY : password);
ftpAuthList.add(ftpAuth);
}
}
if (!smbAuthList.isEmpty()) {
factoryParamMap.put(SmbClient.SMB_AUTHENTICATIONS_PROPERTY, smbAuthList.toArray(new SmbAuthentication[smbAuthList.size()]));
}
if (!smb1AuthList.isEmpty()) {
factoryParamMap.put(org.codelibs.fess.crawler.client.smb1.SmbClient.SMB_AUTHENTICATIONS_PROPERTY, smb1AuthList.toArray(new org.codelibs.fess.crawler.client.smb1.SmbAuthentication[smb1AuthList.size()]));
}
if (!ftpAuthList.isEmpty()) {
factoryParamMap.put(FtpClient.FTP_AUTHENTICATIONS_PROPERTY, ftpAuthList.toArray(new FtpAuthentication[ftpAuthList.size()]));
}
}
crawlerClientFactory = factory;
return factory;
}
use of org.codelibs.fess.crawler.client.ftp.FtpAuthentication in project fess by codelibs.
the class FileConfig method initializeClientFactory.
@Override
public CrawlerClientFactory initializeClientFactory(final Supplier<CrawlerClientFactory> creator) {
if (crawlerClientFactory != null) {
return crawlerClientFactory;
}
final CrawlerClientFactory factory = creator.get();
final FileAuthenticationService fileAuthenticationService = ComponentUtil.getComponent(FileAuthenticationService.class);
// Parameters
final Map<String, Object> paramMap = new HashMap<>();
factory.setInitParameterMap(paramMap);
final Map<String, String> clientConfigMap = getConfigParameterMap(ConfigName.CLIENT);
if (clientConfigMap != null) {
paramMap.putAll(clientConfigMap);
}
// auth params
final List<FileAuthentication> fileAuthList = fileAuthenticationService.getFileAuthenticationList(getId());
final List<SmbAuthentication> smbAuthList = new ArrayList<>();
final List<org.codelibs.fess.crawler.client.smb1.SmbAuthentication> smb1AuthList = new ArrayList<>();
final List<FtpAuthentication> ftpAuthList = new ArrayList<>();
for (final FileAuthentication fileAuth : fileAuthList) {
if (Constants.SAMBA.equals(fileAuth.getProtocolScheme())) {
final SmbAuthentication smbAuth = new SmbAuthentication();
final Map<String, String> map = ParameterUtil.parse(fileAuth.getParameters());
final String domain = map.get("domain");
smbAuth.setDomain(domain == null ? StringUtil.EMPTY : domain);
smbAuth.setServer(fileAuth.getHostname());
smbAuth.setPort(fileAuth.getPort() == null ? -1 : fileAuth.getPort());
smbAuth.setUsername(fileAuth.getUsername());
smbAuth.setPassword(fileAuth.getPassword());
smbAuthList.add(smbAuth);
final org.codelibs.fess.crawler.client.smb1.SmbAuthentication smb1Auth = new org.codelibs.fess.crawler.client.smb1.SmbAuthentication();
smb1Auth.setDomain(domain == null ? StringUtil.EMPTY : domain);
smb1Auth.setServer(fileAuth.getHostname());
smb1Auth.setPort(fileAuth.getPort() == null ? -1 : fileAuth.getPort());
smb1Auth.setUsername(fileAuth.getUsername());
smb1Auth.setPassword(fileAuth.getPassword());
smb1AuthList.add(smb1Auth);
} else if (Constants.FTP.equals(fileAuth.getProtocolScheme())) {
final FtpAuthentication ftpAuth = new FtpAuthentication();
ftpAuth.setServer(fileAuth.getHostname());
ftpAuth.setPort(fileAuth.getPort() == null ? -1 : fileAuth.getPort());
ftpAuth.setUsername(fileAuth.getUsername());
ftpAuth.setPassword(fileAuth.getPassword());
ftpAuthList.add(ftpAuth);
}
}
paramMap.put(Param.Client.SMB_AUTHENTICATIONS, smbAuthList.toArray(new SmbAuthentication[smbAuthList.size()]));
paramMap.put(Param.Client.SMB1_AUTHENTICATIONS, smb1AuthList.toArray(new org.codelibs.fess.crawler.client.smb1.SmbAuthentication[smb1AuthList.size()]));
paramMap.put(Param.Client.FTP_AUTHENTICATIONS, ftpAuthList.toArray(new FtpAuthentication[ftpAuthList.size()]));
crawlerClientFactory = factory;
return factory;
}
Aggregations