Examples with SwiftObjectPath org.apache.hadoop.fs.swift.util.SwiftObjectPath used on opensource projects

Example 1 with SwiftObjectPath

use of org.apache.hadoop.fs.swift.util.SwiftObjectPath in project hadoop by apache.

the class TestSwiftObjectPath method testConvertToPath.

@Test(timeout = SWIFT_TEST_TIMEOUT)
public void testConvertToPath() throws Throwable {
    String initialpath = "/dir/file1";
    Path ipath = new Path(initialpath);
    SwiftObjectPath objectPath = SwiftObjectPath.fromPath(new URI(initialpath), ipath);
    URI endpoint = new URI(ENDPOINT);
    URI uri = SwiftRestClient.pathToURI(objectPath, endpoint);
    LOG.info("Inital Hadoop Path =" + initialpath);
    LOG.info("Merged URI=" + uri);
}

Example 2 with SwiftObjectPath

use of org.apache.hadoop.fs.swift.util.SwiftObjectPath in project hadoop by apache.

the class TestSwiftObjectPath method testChildOfRoot.

@Test(timeout = SWIFT_TEST_TIMEOUT)
public void testChildOfRoot() throws Throwable {
    SwiftObjectPath root = new SwiftObjectPath("container", "/");
    SwiftObjectPath child = new SwiftObjectPath("container", "child");
    SwiftObjectPath grandchild = new SwiftObjectPath("container", "/child/grandchild");
    assertParentOf(root, child);
    assertParentOf(root, grandchild);
    assertParentOf(child, grandchild);
    assertParentOf(root, root);
    assertNotParentOf(child, root);
    assertParentOf(child, child);
    assertNotParentOf(grandchild, root);
}

Example 3 with SwiftObjectPath

use of org.apache.hadoop.fs.swift.util.SwiftObjectPath in project hadoop by apache.

the class TestSwiftObjectPath method testHandleUrlAsPath.

@Test(timeout = SWIFT_TEST_TIMEOUT)
public void testHandleUrlAsPath() throws Exception {
    final String hostPart = "swift://container.service1";
    final String pathPart = "/home/user/files/file1";
    final String uriString = hostPart + pathPart;
    final SwiftObjectPath expected = new SwiftObjectPath(uriString, pathPart);
    final SwiftObjectPath actual = new SwiftObjectPath(uriString, uriString);
    assertEquals(expected, actual);
}

Example 4 with SwiftObjectPath

use of org.apache.hadoop.fs.swift.util.SwiftObjectPath in project hadoop by apache.

the class SwiftNativeFileSystemStore method rename.

/**
   * Rename through copy-and-delete. this is a consequence of the
   * Swift filesystem using the path as the hash
   * into the Distributed Hash Table, "the ring" of filenames.
   * <p>
   * Because of the nature of the operation, it is not atomic.
   *
   * @param src source file/dir
   * @param dst destination
   * @throws IOException                   IO failure
   * @throws SwiftOperationFailedException if the rename failed
   * @throws FileNotFoundException         if the source directory is missing, or
   *                                       the parent directory of the destination
   */
public void rename(Path src, Path dst) throws FileNotFoundException, SwiftOperationFailedException, IOException {
    if (LOG.isDebugEnabled()) {
        LOG.debug("mv " + src + " " + dst);
    }
    boolean renamingOnToSelf = src.equals(dst);
    SwiftObjectPath srcObject = toObjectPath(src);
    SwiftObjectPath destObject = toObjectPath(dst);
    if (SwiftUtils.isRootDir(srcObject)) {
        throw new SwiftOperationFailedException("cannot rename root dir");
    }
    final SwiftFileStatus srcMetadata;
    srcMetadata = getObjectMetadata(src);
    SwiftFileStatus dstMetadata;
    try {
        dstMetadata = getObjectMetadata(dst);
    } catch (FileNotFoundException e) {
        //destination does not exist.
        LOG.debug("Destination does not exist");
        dstMetadata = null;
    }
    //check to see if the destination parent directory exists
    Path srcParent = src.getParent();
    Path dstParent = dst.getParent();
    //directory is root, in which case it must also exist
    if (dstParent != null && !dstParent.equals(srcParent)) {
        try {
            getObjectMetadata(dstParent);
        } catch (FileNotFoundException e) {
            //destination parent doesn't exist; bail out
            LOG.debug("destination parent directory " + dstParent + " doesn't exist");
            throw e;
        }
    }
    boolean destExists = dstMetadata != null;
    boolean destIsDir = destExists && SwiftUtils.isDirectory(dstMetadata);
    //calculate the destination
    SwiftObjectPath destPath;
    //enum the child entries and everything underneath
    List<FileStatus> childStats = listDirectory(srcObject, true, true);
    boolean srcIsFile = !srcMetadata.isDir();
    if (srcIsFile) {
        // #3 dest does not exist: use dest as name
        if (destExists) {
            if (destIsDir) {
                //outcome #2 -move to subdir of dest
                destPath = toObjectPath(new Path(dst, src.getName()));
            } else {
                //outcome #1 dest it's a file: fail if different
                if (!renamingOnToSelf) {
                    throw new FileAlreadyExistsException("cannot rename a file over one that already exists");
                } else {
                    //is mv self self where self is a file. this becomes a no-op
                    LOG.debug("Renaming file onto self: no-op => success");
                    return;
                }
            }
        } else {
            //outcome #3 -new entry
            destPath = toObjectPath(dst);
        }
        int childCount = childStats.size();
        // ->
        if (childCount == 0) {
            copyThenDeleteObject(srcObject, destPath);
        } else {
            //do the copy
            SwiftUtils.debug(LOG, "Source file appears to be partitioned." + " copying file and deleting children");
            copyObject(srcObject, destPath);
            for (FileStatus stat : childStats) {
                SwiftUtils.debug(LOG, "Deleting partitioned file %s ", stat);
                deleteObject(stat.getPath());
            }
            swiftRestClient.delete(srcObject);
        }
    } else {
        if (destExists && !destIsDir) {
            // #1 destination is a file: fail
            throw new FileAlreadyExistsException("the source is a directory, but not the destination");
        }
        Path targetPath;
        if (destExists) {
            // #2 destination is a directory: create a new dir under that one
            targetPath = new Path(dst, src.getName());
        } else {
            // #3 destination doesn't exist: create a new dir with that name
            targetPath = dst;
        }
        SwiftObjectPath targetObjectPath = toObjectPath(targetPath);
        //final check for any recursive operations
        if (srcObject.isEqualToOrParentOf(targetObjectPath)) {
            //you can't rename a directory onto itself
            throw new SwiftOperationFailedException("cannot move a directory under itself");
        }
        LOG.info("mv  " + srcObject + " " + targetPath);
        logDirectory("Directory to copy ", srcObject, childStats);
        // iterative copy of everything under the directory.
        // by listing all children this can be done iteratively
        // rather than recursively -everything in this list is either a file
        // or a 0-byte-len file pretending to be a directory.
        String srcURI = src.toUri().toString();
        int prefixStripCount = srcURI.length() + 1;
        for (FileStatus fileStatus : childStats) {
            Path copySourcePath = fileStatus.getPath();
            String copySourceURI = copySourcePath.toUri().toString();
            String copyDestSubPath = copySourceURI.substring(prefixStripCount);
            Path copyDestPath = new Path(targetPath, copyDestSubPath);
            if (LOG.isTraceEnabled()) {
                //trace to debug some low-level rename path problems; retained
                //in case they ever come back.
                LOG.trace("srcURI=" + srcURI + "; copySourceURI=" + copySourceURI + "; copyDestSubPath=" + copyDestSubPath + "; copyDestPath=" + copyDestPath);
            }
            SwiftObjectPath copyDestination = toObjectPath(copyDestPath);
            try {
                copyThenDeleteObject(toObjectPath(copySourcePath), copyDestination);
            } catch (FileNotFoundException e) {
                LOG.info("Skipping rename of " + copySourcePath);
            }
            //add a throttle delay
            throttle();
        }
        //now rename self. If missing, create the dest directory and warn
        if (!SwiftUtils.isRootDir(srcObject)) {
            try {
                copyThenDeleteObject(srcObject, targetObjectPath);
            } catch (FileNotFoundException e) {
                //create the destination directory
                LOG.warn("Source directory deleted during rename", e);
                innerCreateDirectory(destObject);
            }
        }
    }
}

Example 5 with SwiftObjectPath

use of org.apache.hadoop.fs.swift.util.SwiftObjectPath in project hadoop by apache.

the class SwiftNativeFileSystemStore method uploadFilePart.

/**
   * Upload part of a larger file.
   *
   * @param path        destination path
   * @param partNumber  item number in the path
   * @param inputStream input data
   * @param length      length of the data
   * @throws IOException on a problem
   */
public void uploadFilePart(Path path, int partNumber, InputStream inputStream, long length) throws IOException {
    String stringPath = path.toUri().toString();
    String partitionFilename = SwiftUtils.partitionFilenameFromNumber(partNumber);
    if (stringPath.endsWith("/")) {
        stringPath = stringPath.concat(partitionFilename);
    } else {
        stringPath = stringPath.concat("/").concat(partitionFilename);
    }
    swiftRestClient.upload(new SwiftObjectPath(toDirPath(path).getContainer(), stringPath), inputStream, length);
}