Search in sources :

Example 1 with SwiftConnectionManager

use of com.ibm.stocator.fs.swift.http.SwiftConnectionManager in project stocator by SparkTC.

the class SwiftAPIClient method initiate.

@Override
public void initiate(String scheme) throws IOException, ConfigurationParseException {
    cachedSparkOriginated = new HashMap<String, Boolean>();
    cachedSparkJobsStatus = new HashMap<String, Boolean>();
    schemaProvided = scheme;
    Properties props = ConfigurationHandler.initialize(filesystemURI, conf);
    connectionConfiguration.setExecutionCount(conf.getInt(Constants.EXECUTION_RETRY, ConnectionConfiguration.DEFAULT_EXECUTION_RETRY));
    connectionConfiguration.setMaxPerRoute(conf.getInt(Constants.MAX_PER_ROUTE, ConnectionConfiguration.DEFAULT_MAX_PER_ROUTE));
    connectionConfiguration.setMaxTotal(conf.getInt(Constants.MAX_TOTAL_CONNECTIONS, ConnectionConfiguration.DEFAULT_MAX_TOTAL_CONNECTIONS));
    connectionConfiguration.setReqConnectionRequestTimeout(conf.getInt(Constants.REQUEST_CONNECTION_TIMEOUT, ConnectionConfiguration.DEFAULT_REQUEST_CONNECTION_TIMEOUT));
    connectionConfiguration.setReqConnectTimeout(conf.getInt(Constants.REQUEST_CONNECT_TIMEOUT, ConnectionConfiguration.DEFAULT_REQUEST_CONNECT_TIMEOUT));
    connectionConfiguration.setReqSocketTimeout(conf.getInt(Constants.REQUEST_SOCKET_TIMEOUT, ConnectionConfiguration.DEFAULT_REQUEST_SOCKET_TIMEOUT));
    connectionConfiguration.setSoTimeout(conf.getInt(Constants.SOCKET_TIMEOUT, ConnectionConfiguration.DEFAULT_SOCKET_TIMEOUT));
    LOG.trace("{} set connection manager", filesystemURI.toString());
    swiftConnectionManager = new SwiftConnectionManager(connectionConfiguration);
    LOG.trace("{}", connectionConfiguration.toString());
    bufferDir = props.getProperty(BUFFER_DIR_PROPERTY, "");
    nonStreamingUpload = "true".equals(props.getProperty(NON_STREAMING_UPLOAD_PROPERTY, "false"));
    AccountConfig config = new AccountConfig();
    fModeAutomaticDelete = "true".equals(props.getProperty(FMODE_AUTOMATIC_DELETE_PROPERTY, "false"));
    blockSize = Long.valueOf(props.getProperty(SWIFT_BLOCK_SIZE_PROPERTY, "128")).longValue() * 1024 * 1024L;
    String authMethod = props.getProperty(SWIFT_AUTH_METHOD_PROPERTY);
    ObjectMapper mapper = new ObjectMapper();
    mapper.configure(SerializationConfig.Feature.WRAP_ROOT_VALUE, true);
    boolean syncWithServer = conf.getBoolean(Constants.JOSS_SYNC_SERVER_TIME, false);
    if (!syncWithServer) {
        LOG.trace("JOSS: disable sync time with server");
        config.setAllowSynchronizeWithServer(false);
    }
    if (authMethod.equals(PUBLIC_ACCESS)) {
        // we need to extract container name and path from the public URL
        String publicURL = filesystemURI.toString().replace(schemaProvided, "https");
        publicContainer = true;
        LOG.debug("publicURL: {}", publicURL);
        String accessURL = Utils.extractAccessURL(publicURL, scheme);
        LOG.debug("auth url {}", accessURL);
        config.setAuthUrl(accessURL);
        config.setAuthenticationMethod(AuthenticationMethod.EXTERNAL);
        container = Utils.extractDataRoot(publicURL, accessURL);
        DummyAccessProvider p = new DummyAccessProvider(accessURL);
        config.setAccessProvider(p);
        mJossAccount = new JossAccount(config, null, true, swiftConnectionManager);
        mJossAccount.createDummyAccount();
    } else {
        container = props.getProperty(SWIFT_CONTAINER_PROPERTY);
        String isPubProp = props.getProperty(SWIFT_PUBLIC_PROPERTY, "false");
        usePublicURL = "true".equals(isPubProp);
        LOG.trace("Use public key value is {}. Use public {}", isPubProp, usePublicURL);
        config.setPassword(props.getProperty(SWIFT_PASSWORD_PROPERTY));
        config.setAuthUrl(Utils.getOption(props, SWIFT_AUTH_PROPERTY));
        if (authMethod.equals("keystone")) {
            preferredRegion = props.getProperty(SWIFT_REGION_PROPERTY);
            if (preferredRegion != null) {
                config.setPreferredRegion(preferredRegion);
            }
            config.setAuthenticationMethod(AuthenticationMethod.KEYSTONE);
            config.setUsername(Utils.getOption(props, SWIFT_USERNAME_PROPERTY));
            config.setTenantName(props.getProperty(SWIFT_TENANT_PROPERTY));
        } else if (authMethod.equals(KEYSTONE_V3_AUTH)) {
            preferredRegion = props.getProperty(SWIFT_REGION_PROPERTY, "dallas");
            config.setPreferredRegion(preferredRegion);
            config.setAuthenticationMethod(AuthenticationMethod.EXTERNAL);
            String userId = props.getProperty(SWIFT_USER_ID_PROPERTY);
            String projectId = props.getProperty(SWIFT_PROJECT_ID_PROPERTY);
            PasswordScopeAccessProvider psap = new PasswordScopeAccessProvider(userId, config.getPassword(), projectId, config.getAuthUrl(), preferredRegion);
            config.setAccessProvider(psap);
        } else if (authMethod.equals("basic")) {
            config.setAuthenticationMethod(AuthenticationMethod.BASIC);
            config.setUsername(Utils.getOption(props, SWIFT_USERNAME_PROPERTY));
        } else {
            config.setAuthenticationMethod(AuthenticationMethod.TEMPAUTH);
            config.setTenantName(Utils.getOption(props, SWIFT_USERNAME_PROPERTY));
            config.setUsername(props.getProperty(SWIFT_TENANT_PROPERTY));
        }
        LOG.trace("{}", config.toString());
        mJossAccount = new JossAccount(config, preferredRegion, usePublicURL, swiftConnectionManager);
        try {
            mJossAccount.createAccount();
        } catch (Exception e) {
            throw new IOException("Failed to create an account model." + " Please check the provided access credentials." + " Verify the validitiy of the auth url: " + config.getAuthUrl(), e);
        }
    }
    Container containerObj = mJossAccount.getAccount().getContainer(container);
    if (!authMethod.equals(PUBLIC_ACCESS) && !containerObj.exists()) {
        try {
            containerObj.create();
        } catch (AlreadyExistsException e) {
            LOG.debug("Create container failed. {} was already exists. ", container);
        }
    }
    objectCache = new SwiftObjectCache(containerObj);
}
Also used : AlreadyExistsException(org.javaswift.joss.exception.AlreadyExistsException) DummyAccessProvider(com.ibm.stocator.fs.swift.auth.DummyAccessProvider) PasswordScopeAccessProvider(com.ibm.stocator.fs.swift.auth.PasswordScopeAccessProvider) IOException(java.io.IOException) Properties(java.util.Properties) SwiftConnectionManager(com.ibm.stocator.fs.swift.http.SwiftConnectionManager) JossAccount(com.ibm.stocator.fs.swift.auth.JossAccount) ConfigurationParseException(com.ibm.stocator.fs.common.exception.ConfigurationParseException) FileNotFoundException(java.io.FileNotFoundException) UnsupportedEncodingException(java.io.UnsupportedEncodingException) AlreadyExistsException(org.javaswift.joss.exception.AlreadyExistsException) IOException(java.io.IOException) AccountConfig(org.javaswift.joss.client.factory.AccountConfig) Container(org.javaswift.joss.model.Container) ObjectMapper(org.codehaus.jackson.map.ObjectMapper)

Aggregations

ConfigurationParseException (com.ibm.stocator.fs.common.exception.ConfigurationParseException)1 DummyAccessProvider (com.ibm.stocator.fs.swift.auth.DummyAccessProvider)1 JossAccount (com.ibm.stocator.fs.swift.auth.JossAccount)1 PasswordScopeAccessProvider (com.ibm.stocator.fs.swift.auth.PasswordScopeAccessProvider)1 SwiftConnectionManager (com.ibm.stocator.fs.swift.http.SwiftConnectionManager)1 FileNotFoundException (java.io.FileNotFoundException)1 IOException (java.io.IOException)1 UnsupportedEncodingException (java.io.UnsupportedEncodingException)1 Properties (java.util.Properties)1 ObjectMapper (org.codehaus.jackson.map.ObjectMapper)1 AccountConfig (org.javaswift.joss.client.factory.AccountConfig)1 AlreadyExistsException (org.javaswift.joss.exception.AlreadyExistsException)1 Container (org.javaswift.joss.model.Container)1