Examples with SwiftConnectionManager com.ibm.stocator.fs.swift.http.SwiftConnectionManager used on opensource projects

Example 1 with SwiftConnectionManager

use of com.ibm.stocator.fs.swift.http.SwiftConnectionManager in project stocator by SparkTC.

the class SwiftAPIClient method initiate.

@Override
public void initiate(String scheme) throws IOException, ConfigurationParseException {
    cachedSparkOriginated = new HashMap<String, Boolean>();
    cachedSparkJobsStatus = new HashMap<String, Boolean>();
    schemaProvided = scheme;
    Properties props = ConfigurationHandler.initialize(filesystemURI, conf);
    connectionConfiguration.setExecutionCount(conf.getInt(Constants.EXECUTION_RETRY, ConnectionConfiguration.DEFAULT_EXECUTION_RETRY));
    connectionConfiguration.setMaxPerRoute(conf.getInt(Constants.MAX_PER_ROUTE, ConnectionConfiguration.DEFAULT_MAX_PER_ROUTE));
    connectionConfiguration.setMaxTotal(conf.getInt(Constants.MAX_TOTAL_CONNECTIONS, ConnectionConfiguration.DEFAULT_MAX_TOTAL_CONNECTIONS));
    connectionConfiguration.setReqConnectionRequestTimeout(conf.getInt(Constants.REQUEST_CONNECTION_TIMEOUT, ConnectionConfiguration.DEFAULT_REQUEST_CONNECTION_TIMEOUT));
    connectionConfiguration.setReqConnectTimeout(conf.getInt(Constants.REQUEST_CONNECT_TIMEOUT, ConnectionConfiguration.DEFAULT_REQUEST_CONNECT_TIMEOUT));
    connectionConfiguration.setReqSocketTimeout(conf.getInt(Constants.REQUEST_SOCKET_TIMEOUT, ConnectionConfiguration.DEFAULT_REQUEST_SOCKET_TIMEOUT));
    connectionConfiguration.setSoTimeout(conf.getInt(Constants.SOCKET_TIMEOUT, ConnectionConfiguration.DEFAULT_SOCKET_TIMEOUT));
    LOG.trace("{} set connection manager", filesystemURI.toString());
    swiftConnectionManager = new SwiftConnectionManager(connectionConfiguration);
    LOG.trace("{}", connectionConfiguration.toString());
    bufferDir = props.getProperty(BUFFER_DIR_PROPERTY, "");
    nonStreamingUpload = "true".equals(props.getProperty(NON_STREAMING_UPLOAD_PROPERTY, "false"));
    AccountConfig config = new AccountConfig();
    fModeAutomaticDelete = "true".equals(props.getProperty(FMODE_AUTOMATIC_DELETE_PROPERTY, "false"));
    blockSize = Long.valueOf(props.getProperty(SWIFT_BLOCK_SIZE_PROPERTY, "128")).longValue() * 1024 * 1024L;
    String authMethod = props.getProperty(SWIFT_AUTH_METHOD_PROPERTY);
    ObjectMapper mapper = new ObjectMapper();
    mapper.configure(SerializationConfig.Feature.WRAP_ROOT_VALUE, true);
    boolean syncWithServer = conf.getBoolean(Constants.JOSS_SYNC_SERVER_TIME, false);
    if (!syncWithServer) {
        LOG.trace("JOSS: disable sync time with server");
        config.setAllowSynchronizeWithServer(false);
    }
    if (authMethod.equals(PUBLIC_ACCESS)) {
        // we need to extract container name and path from the public URL
        String publicURL = filesystemURI.toString().replace(schemaProvided, "https");
        publicContainer = true;
        LOG.debug("publicURL: {}", publicURL);
        String accessURL = Utils.extractAccessURL(publicURL, scheme);
        LOG.debug("auth url {}", accessURL);
        config.setAuthUrl(accessURL);
        config.setAuthenticationMethod(AuthenticationMethod.EXTERNAL);
        container = Utils.extractDataRoot(publicURL, accessURL);
        DummyAccessProvider p = new DummyAccessProvider(accessURL);
        config.setAccessProvider(p);
        mJossAccount = new JossAccount(config, null, true, swiftConnectionManager);
        mJossAccount.createDummyAccount();
    } else {
        container = props.getProperty(SWIFT_CONTAINER_PROPERTY);
        String isPubProp = props.getProperty(SWIFT_PUBLIC_PROPERTY, "false");
        usePublicURL = "true".equals(isPubProp);
        LOG.trace("Use public key value is {}. Use public {}", isPubProp, usePublicURL);
        config.setPassword(props.getProperty(SWIFT_PASSWORD_PROPERTY));
        config.setAuthUrl(Utils.getOption(props, SWIFT_AUTH_PROPERTY));
        if (authMethod.equals("keystone")) {
            preferredRegion = props.getProperty(SWIFT_REGION_PROPERTY);
            if (preferredRegion != null) {
                config.setPreferredRegion(preferredRegion);
            }
            config.setAuthenticationMethod(AuthenticationMethod.KEYSTONE);
            config.setUsername(Utils.getOption(props, SWIFT_USERNAME_PROPERTY));
            config.setTenantName(props.getProperty(SWIFT_TENANT_PROPERTY));
        } else if (authMethod.equals(KEYSTONE_V3_AUTH)) {
            preferredRegion = props.getProperty(SWIFT_REGION_PROPERTY, "dallas");
            config.setPreferredRegion(preferredRegion);
            config.setAuthenticationMethod(AuthenticationMethod.EXTERNAL);
            String userId = props.getProperty(SWIFT_USER_ID_PROPERTY);
            String projectId = props.getProperty(SWIFT_PROJECT_ID_PROPERTY);
            PasswordScopeAccessProvider psap = new PasswordScopeAccessProvider(userId, config.getPassword(), projectId, config.getAuthUrl(), preferredRegion);
            config.setAccessProvider(psap);
        } else if (authMethod.equals("basic")) {
            config.setAuthenticationMethod(AuthenticationMethod.BASIC);
            config.setUsername(Utils.getOption(props, SWIFT_USERNAME_PROPERTY));
        } else {
            config.setAuthenticationMethod(AuthenticationMethod.TEMPAUTH);
            config.setTenantName(Utils.getOption(props, SWIFT_USERNAME_PROPERTY));
            config.setUsername(props.getProperty(SWIFT_TENANT_PROPERTY));
        }
        LOG.trace("{}", config.toString());
        mJossAccount = new JossAccount(config, preferredRegion, usePublicURL, swiftConnectionManager);
        try {
            mJossAccount.createAccount();
        } catch (Exception e) {
            throw new IOException("Failed to create an account model." + " Please check the provided access credentials." + " Verify the validitiy of the auth url: " + config.getAuthUrl(), e);
        }
    }
    Container containerObj = mJossAccount.getAccount().getContainer(container);
    if (!authMethod.equals(PUBLIC_ACCESS) && !containerObj.exists()) {
        try {
            containerObj.create();
        } catch (AlreadyExistsException e) {
            LOG.debug("Create container failed. {} was already exists. ", container);
        }
    }
    objectCache = new SwiftObjectCache(containerObj);
}