Examples with Container org.javaswift.joss.model.Container used on opensource projects

Example 6 with Container

use of org.javaswift.joss.model.Container in project alluxio by Alluxio.

the class SwiftMockOutputStream method close.

@Override
public void close() throws IOException {
    if (mClosed) {
        return;
    }
    mOutputStream.close();
    try {
        Container container = mAccount.getContainer(mContainerName);
        StoredObject object = container.getObject(mObjectName);
        object.uploadObject(mFile);
    } catch (Exception e) {
        throw new IOException(e);
    }
    // Close successful
    mClosed = true;
}

Example 7 with Container

use of org.javaswift.joss.model.Container in project stocator by SparkTC.

the class SwiftAPIClient method initiate.

@Override
public void initiate(String scheme) throws IOException, ConfigurationParseException {
    cachedSparkOriginated = new HashMap<String, Boolean>();
    cachedSparkJobsStatus = new HashMap<String, Boolean>();
    schemaProvided = scheme;
    Properties props = ConfigurationHandler.initialize(filesystemURI, conf);
    connectionConfiguration.setExecutionCount(conf.getInt(Constants.EXECUTION_RETRY, ConnectionConfiguration.DEFAULT_EXECUTION_RETRY));
    connectionConfiguration.setMaxPerRoute(conf.getInt(Constants.MAX_PER_ROUTE, ConnectionConfiguration.DEFAULT_MAX_PER_ROUTE));
    connectionConfiguration.setMaxTotal(conf.getInt(Constants.MAX_TOTAL_CONNECTIONS, ConnectionConfiguration.DEFAULT_MAX_TOTAL_CONNECTIONS));
    connectionConfiguration.setReqConnectionRequestTimeout(conf.getInt(Constants.REQUEST_CONNECTION_TIMEOUT, ConnectionConfiguration.DEFAULT_REQUEST_CONNECTION_TIMEOUT));
    connectionConfiguration.setReqConnectTimeout(conf.getInt(Constants.REQUEST_CONNECT_TIMEOUT, ConnectionConfiguration.DEFAULT_REQUEST_CONNECT_TIMEOUT));
    connectionConfiguration.setReqSocketTimeout(conf.getInt(Constants.REQUEST_SOCKET_TIMEOUT, ConnectionConfiguration.DEFAULT_REQUEST_SOCKET_TIMEOUT));
    connectionConfiguration.setSoTimeout(conf.getInt(Constants.SOCKET_TIMEOUT, ConnectionConfiguration.DEFAULT_SOCKET_TIMEOUT));
    LOG.trace("{} set connection manager", filesystemURI.toString());
    swiftConnectionManager = new SwiftConnectionManager(connectionConfiguration);
    LOG.trace("{}", connectionConfiguration.toString());
    bufferDir = props.getProperty(BUFFER_DIR_PROPERTY, "");
    nonStreamingUpload = "true".equals(props.getProperty(NON_STREAMING_UPLOAD_PROPERTY, "false"));
    AccountConfig config = new AccountConfig();
    fModeAutomaticDelete = "true".equals(props.getProperty(FMODE_AUTOMATIC_DELETE_PROPERTY, "false"));
    blockSize = Long.valueOf(props.getProperty(SWIFT_BLOCK_SIZE_PROPERTY, "128")).longValue() * 1024 * 1024L;
    String authMethod = props.getProperty(SWIFT_AUTH_METHOD_PROPERTY);
    ObjectMapper mapper = new ObjectMapper();
    mapper.configure(SerializationConfig.Feature.WRAP_ROOT_VALUE, true);
    boolean syncWithServer = conf.getBoolean(Constants.JOSS_SYNC_SERVER_TIME, false);
    if (!syncWithServer) {
        LOG.trace("JOSS: disable sync time with server");
        config.setAllowSynchronizeWithServer(false);
    }
    if (authMethod.equals(PUBLIC_ACCESS)) {
        // we need to extract container name and path from the public URL
        String publicURL = filesystemURI.toString().replace(schemaProvided, "https");
        publicContainer = true;
        LOG.debug("publicURL: {}", publicURL);
        String accessURL = Utils.extractAccessURL(publicURL, scheme);
        LOG.debug("auth url {}", accessURL);
        config.setAuthUrl(accessURL);
        config.setAuthenticationMethod(AuthenticationMethod.EXTERNAL);
        container = Utils.extractDataRoot(publicURL, accessURL);
        DummyAccessProvider p = new DummyAccessProvider(accessURL);
        config.setAccessProvider(p);
        mJossAccount = new JossAccount(config, null, true, swiftConnectionManager);
        mJossAccount.createDummyAccount();
    } else {
        container = props.getProperty(SWIFT_CONTAINER_PROPERTY);
        String isPubProp = props.getProperty(SWIFT_PUBLIC_PROPERTY, "false");
        usePublicURL = "true".equals(isPubProp);
        LOG.trace("Use public key value is {}. Use public {}", isPubProp, usePublicURL);
        config.setPassword(props.getProperty(SWIFT_PASSWORD_PROPERTY));
        config.setAuthUrl(Utils.getOption(props, SWIFT_AUTH_PROPERTY));
        if (authMethod.equals("keystone")) {
            preferredRegion = props.getProperty(SWIFT_REGION_PROPERTY);
            if (preferredRegion != null) {
                config.setPreferredRegion(preferredRegion);
            }
            config.setAuthenticationMethod(AuthenticationMethod.KEYSTONE);
            config.setUsername(Utils.getOption(props, SWIFT_USERNAME_PROPERTY));
            config.setTenantName(props.getProperty(SWIFT_TENANT_PROPERTY));
        } else if (authMethod.equals(KEYSTONE_V3_AUTH)) {
            preferredRegion = props.getProperty(SWIFT_REGION_PROPERTY, "dallas");
            config.setPreferredRegion(preferredRegion);
            config.setAuthenticationMethod(AuthenticationMethod.EXTERNAL);
            String userId = props.getProperty(SWIFT_USER_ID_PROPERTY);
            String projectId = props.getProperty(SWIFT_PROJECT_ID_PROPERTY);
            PasswordScopeAccessProvider psap = new PasswordScopeAccessProvider(userId, config.getPassword(), projectId, config.getAuthUrl(), preferredRegion);
            config.setAccessProvider(psap);
        } else if (authMethod.equals("basic")) {
            config.setAuthenticationMethod(AuthenticationMethod.BASIC);
            config.setUsername(Utils.getOption(props, SWIFT_USERNAME_PROPERTY));
        } else {
            config.setAuthenticationMethod(AuthenticationMethod.TEMPAUTH);
            config.setTenantName(Utils.getOption(props, SWIFT_USERNAME_PROPERTY));
            config.setUsername(props.getProperty(SWIFT_TENANT_PROPERTY));
        }
        LOG.trace("{}", config.toString());
        mJossAccount = new JossAccount(config, preferredRegion, usePublicURL, swiftConnectionManager);
        try {
            mJossAccount.createAccount();
        } catch (Exception e) {
            throw new IOException("Failed to create an account model." + " Please check the provided access credentials." + " Verify the validitiy of the auth url: " + config.getAuthUrl(), e);
        }
    }
    Container containerObj = mJossAccount.getAccount().getContainer(container);
    if (!authMethod.equals(PUBLIC_ACCESS) && !containerObj.exists()) {
        try {
            containerObj.create();
        } catch (AlreadyExistsException e) {
            LOG.debug("Create container failed. {} was already exists. ", container);
        }
    }
    objectCache = new SwiftObjectCache(containerObj);
}

Example 8 with Container

use of org.javaswift.joss.model.Container in project stocator by SparkTC.

the class SwiftAPIClient method rename.

@Override
public boolean rename(String hostName, String srcPath, String dstPath) throws IOException {
    LOG.debug("Rename from {} to {}. hostname is {}", srcPath, dstPath, hostName);
    String objNameSrc = srcPath.toString();
    if (srcPath.toString().startsWith(hostName)) {
        objNameSrc = getObjName(hostName, srcPath);
    }
    String objNameDst = dstPath.toString();
    if (objNameDst.toString().startsWith(hostName)) {
        objNameDst = getObjName(hostName, dstPath);
    }
    if (stocatorPath.isTemporaryPathContain(objNameSrc)) {
        LOG.debug("Rename on the temp object {}. Return true", objNameSrc);
        return true;
    }
    LOG.debug("Rename modified from {} to {}", objNameSrc, objNameDst);
    Container cont = mJossAccount.getAccount().getContainer(container);
    StoredObject so = cont.getObject(objNameSrc);
    StoredObject soDst = cont.getObject(objNameDst);
    so.copyObject(cont, soDst);
    return true;
}

Example 9 with Container

use of org.javaswift.joss.model.Container in project stocator by SparkTC.

the class SwiftAPIClient method getFileStatus.

@Override
public FileStatus getFileStatus(String hostName, Path path, String msg) throws IOException, FileNotFoundException {
    LOG.trace("Get object metadata ({}): {}, hostname: {}", msg, path, hostName);
    Container cont = mJossAccount.getAccount().getContainer(container);
    /*
     * Check if the path is a temporary URL
     */
    if (path.toString().contains("temp_url")) {
        long length = SwiftAPIDirect.getTempUrlObjectLength(path, swiftConnectionManager);
        return new FileStatus(length, false, 1, blockSize, 0L, path);
    }
    /*
      The requested path is equal to hostName.
      HostName is equal to hostNameScheme, thus the container.
      Therefore we have no object to look for and
      we return the FileStatus as a directory.
      Containers have to lastModified.
     */
    if (path.toString().equals(hostName) || (path.toString().length() + 1 == hostName.length())) {
        LOG.debug("{}: metadata requested on container", path.toString());
        return new FileStatus(0L, true, 1, blockSize, 0L, path);
    }
    /*
      The requested path is not equal to the container.
      We need to check if the object requested is a real object or a directory.
      This may be triggered when users want to access a directory rather than
      the entire container.
      A directory in Swift can have two implementations:
      1) a zero byte object with the name of the directory
      2) no zero byte object with the name of the directory
    */
    boolean isDirectory = false;
    String objectName = getObjName(hostName, path);
    if (stocatorPath.isTemporaryPathContain(objectName)) {
        LOG.debug("getObjectMetadata on temp object {}. Return not found", objectName);
        throw new FileNotFoundException("Not found " + path.toString());
    }
    SwiftCachedObject obj = objectCache.get(objectName);
    if (obj != null) {
        // If so, it might be a directory
        if (obj.getContentLength() == 0) {
            Collection<DirectoryOrObject> directoryFiles = cont.listDirectory(objectName, '/', "", 10);
            if (directoryFiles != null && directoryFiles.size() != 0) {
                // The zero length object is a directory
                isDirectory = true;
            }
        }
        LOG.trace("{} is object. isDirectory: {}  lastModified: {}", path.toString(), isDirectory, obj.getLastModified());
        return new FileStatus(obj.getContentLength(), isDirectory, 1, blockSize, obj.getLastModified(), path);
    }
    // We need to check if it may be a directory with no zero byte file associated
    LOG.trace("Checking if directory without 0 byte object associated {}", objectName);
    Collection<DirectoryOrObject> directoryFiles = cont.listDirectory(objectName + "/", '/', "", 10);
    if (directoryFiles != null) {
        LOG.trace("{} got {} candidates", objectName + "/", directoryFiles.size());
    }
    if (directoryFiles != null && directoryFiles.size() != 0) {
        // In this case there is no lastModified
        isDirectory = true;
        LOG.debug("Got object {}. isDirectory: {}  lastModified: {}", path, isDirectory, null);
        return new FileStatus(0, isDirectory, 1, blockSize, 0L, path);
    }
    LOG.debug("Not found {}", path.toString());
    throw new FileNotFoundException("No such object exists " + path.toString());
}

Example 10 with Container

use of org.javaswift.joss.model.Container in project stocator by SparkTC.

the class SwiftAPIClient method list.

/**
 * {@inheritDoc}
 *
 * some examples of failed attempts:
 * a/b/c.data/part-00099-attempt_201603171503_0001_m_000099_119
 * a/b/c.data/part-00099-attempt_201603171503_0001_m_000099_120
 * a/b/c.data/part-00099-attempt_201603171503_0001_m_000099_121
 * a/b/c.data/part-00099-attempt_201603171503_0001_m_000099_122
 * or
 * a/b/c.data/part-r-00000-48ae3461-203f-4dd3-b141-a45426e2d26c
 * .csv-attempt_201603171328_0000_m_000000_1
 * a/b/c.data/part-r-00000-48ae3461-203f-4dd3-b141-a45426e2d26c
 * .csv-attempt_201603171328_0000_m_000000_0
 * in all the cases format is objectname-taskid where
 * taskid may vary, depends how many tasks were re-submitted
 *
 * @param hostName hostname
 * @param path path to the object
 * @param fullListing if true, will return objects of size 0
 * @param prefixBased if set to true, container will be listed with prefix based query
 * @return Array of Hadoop FileStatus
 * @throws IOException in case of network failure
 */
public FileStatus[] list(String hostName, Path path, boolean fullListing, boolean prefixBased, Boolean isDirectory, boolean flatListing, PathFilter filter) throws IOException {
    LOG.debug("List container: raw path parent {} container {} hostname {}", path.toString(), container, hostName);
    Container cObj = mJossAccount.getAccount().getContainer(container);
    String obj;
    if (path.toString().equals(container) || publicContainer) {
        obj = "";
    } else if (path.toString().startsWith(container + "/")) {
        obj = path.toString().substring(container.length() + 1);
    } else if (path.toString().startsWith(hostName)) {
        obj = path.toString().substring(hostName.length());
    } else {
        obj = path.toString();
    }
    LOG.debug("List container for {} container {}", obj, container);
    ArrayList<FileStatus> tmpResult = new ArrayList<FileStatus>();
    StoredObject previousElement = null;
    boolean moreData = true;
    String marker = null;
    FileStatus fs = null;
    while (moreData) {
        Collection<StoredObject> res = cObj.list(obj, marker, pageListSize);
        moreData = (res.size() == pageListSize);
        if (marker == null && (res == null || res.isEmpty() || res.size() == 0)) {
            FileStatus[] emptyRes = {};
            LOG.debug("List {} in container {} is empty", obj, container);
            return emptyRes;
        }
        for (StoredObject tmp : res) {
            marker = tmp.getAsObject().getName();
            if (previousElement == null) {
                // first entry
                setCorrectSize(tmp, cObj);
                previousElement = tmp.getAsObject();
                continue;
            }
            String unifiedObjectName = extractUnifiedObjectName(tmp.getName());
            LOG.trace("{} Matching {}", unifiedObjectName, obj);
            if (!prefixBased && !obj.equals("") && !path.toString().endsWith("/") && !unifiedObjectName.equals(obj) && !unifiedObjectName.startsWith(obj + "/")) {
                // JOSS returns all objects that start with the prefix of obj.
                // These may include other unrelated objects.
                LOG.trace("{} does not match {}. Skipped", unifiedObjectName, obj);
                continue;
            } else if (isDirectory && !unifiedObjectName.equals(obj) && !unifiedObjectName.startsWith(obj + "/")) {
                LOG.trace("directory {}. {} does not match {}. Skipped", isDirectory, unifiedObjectName, obj);
                continue;
            }
            LOG.trace("Unified name: {}, path {}", unifiedObjectName, tmp.getName());
            if (!unifiedObjectName.equals(tmp.getName()) && isSparkOrigin(unifiedObjectName) && !fullListing) {
                LOG.trace("{} created by Spark", unifiedObjectName);
                if (!isJobSuccessful(unifiedObjectName)) {
                    LOG.trace("{} created by failed Spark job. Skipped", unifiedObjectName);
                    if (fModeAutomaticDelete) {
                        delete(hostName, new Path(tmp.getName()), true);
                    }
                    continue;
                } else {
                    // we need to make sure there are no failed attempts
                    if (nameWithoutTaskID(tmp.getName()).equals(nameWithoutTaskID(previousElement.getName()))) {
                        // found failed that was not aborted.
                        LOG.trace("Colision identified between {} and {}", previousElement.getName(), tmp.getName());
                        setCorrectSize(tmp, cObj);
                        if (previousElement.getContentLength() < tmp.getContentLength()) {
                            LOG.trace("New candidate is {}. Removed {}", tmp.getName(), previousElement.getName());
                            previousElement = tmp.getAsObject();
                        }
                        continue;
                    }
                }
            }
            fs = null;
            if (previousElement.getContentLength() > 0 || fullListing) {
                fs = createFileStatus(previousElement, cObj, hostName, path);
                objectCache.put(getObjName(hostName, fs.getPath()), fs.getLen(), fs.getModificationTime());
                tmpResult.add(fs);
            }
            previousElement = tmp.getAsObject();
        }
    }
    if (previousElement != null && (previousElement.getContentLength() > 0 || fullListing)) {
        LOG.trace("Adding {} to the list", previousElement.getPath());
        fs = createFileStatus(previousElement, cObj, hostName, path);
        if (filter == null) {
            objectCache.put(getObjName(hostName, fs.getPath()), fs.getLen(), fs.getModificationTime());
            tmpResult.add(fs);
        } else if (filter != null && filter.accept(fs.getPath())) {
            objectCache.put(getObjName(hostName, fs.getPath()), fs.getLen(), fs.getModificationTime());
            tmpResult.add(fs);
        } else {
            LOG.trace("{} rejected by path filter during list", fs.getPath());
        }
    }
    LOG.debug("Listing of {} completed with {} results", path.toString(), tmpResult.size());
    return tmpResult.toArray(new FileStatus[tmpResult.size()]);
}