Examples with IgfsPath org.apache.ignite.igfs.IgfsPath used on opensource projects

Example 86 with IgfsPath

use of org.apache.ignite.igfs.IgfsPath in project ignite by apache.

the class IgfsMetaManager method move.

/**
 * Move routine.
 *
 * @param srcPath Source path.
 * @param dstPath Destination path.
 * @throws IgniteCheckedException In case of exception.
 */
public void move(IgfsPath srcPath, IgfsPath dstPath) throws IgniteCheckedException {
    if (busyLock.enterBusy()) {
        try {
            validTxState(false);
            // Prepare path IDs.
            IgfsPathIds srcPathIds = pathIds(srcPath);
            IgfsPathIds dstPathIds = pathIds(dstPath);
            // Source path must exists.
            if (!srcPathIds.allExists())
                throw new IgfsPathNotFoundException("Failed to perform move because source path is not " + "found: " + srcPath);
            // At this point we need to understand name of resulting entry. It will be either destination leaf
            // or source leaf depending on existence.
            String dstName;
            if (dstPathIds.lastExists())
                // Full destination path exists -> use source name.
                dstName = srcPathIds.lastPart();
            else {
                if (dstPathIds.lastParentExists()) {
                    // Destination path doesn't exists -> use destination name.
                    dstName = dstPathIds.lastPart();
                    dstPathIds = dstPathIds.parent();
                } else
                    // Destination parent is not found either -> exception.
                    throw new IgfsPathNotFoundException("Failed to perform move because destination path is not " + "found: " + dstPath.parent());
            }
            // Lock participating IDs.
            final Set<IgniteUuid> lockIds = new TreeSet<>(PATH_ID_SORTING_COMPARATOR);
            srcPathIds.addExistingIds(lockIds, relaxed);
            dstPathIds.addExistingIds(lockIds, relaxed);
            try (GridNearTxLocal tx = startTx()) {
                // Obtain the locks.
                final Map<IgniteUuid, IgfsEntryInfo> lockInfos = lockIds(lockIds);
                // Verify integrity of source and destination paths.
                if (!srcPathIds.verifyIntegrity(lockInfos, relaxed))
                    throw new IgfsPathNotFoundException("Failed to perform move because source directory " + "structure changed concurrently [src=" + srcPath + ", dst=" + dstPath + ']');
                if (!dstPathIds.verifyIntegrity(lockInfos, relaxed))
                    throw new IgfsPathNotFoundException("Failed to perform move because destination directory " + "structure changed concurrently [src=" + srcPath + ", dst=" + dstPath + ']');
                // Addiional check: is destination directory?
                IgfsEntryInfo dstParentInfo = lockInfos.get(dstPathIds.lastId());
                if (dstParentInfo.isFile())
                    throw new IgfsPathAlreadyExistsException("Failed to perform move because destination points " + "to existing file [src=" + srcPath + ", dst=" + dstPath + ']');
                // Additional check: does destination already has child with the same name?
                if (dstParentInfo.hasChild(dstName))
                    throw new IgfsPathAlreadyExistsException("Failed to perform move because destination already " + "contains entry with the same name existing file [src=" + srcPath + ", dst=" + dstPath + ']');
                // Actual move: remove from source parent and add to destination target.
                IgfsEntryInfo srcParentInfo = lockInfos.get(srcPathIds.lastParentId());
                IgfsEntryInfo srcInfo = lockInfos.get(srcPathIds.lastId());
                String srcName = srcPathIds.lastPart();
                IgfsListingEntry srcEntry = srcParentInfo.listing().get(srcName);
                transferEntry(srcEntry, srcParentInfo.id(), srcName, dstParentInfo.id(), dstName);
                tx.commit();
                // Fire events.
                IgfsPath newPath = new IgfsPath(dstPathIds.path(), dstName);
                IgfsUtils.sendEvents(igfsCtx.kernalContext(), srcPath, newPath, srcInfo.isFile() ? EVT_IGFS_FILE_RENAMED : EVT_IGFS_DIR_RENAMED);
            }
        } finally {
            busyLock.leaveBusy();
        }
    } else
        throw new IllegalStateException("Failed to perform move because Grid is stopping [srcPath=" + srcPath + ", dstPath=" + dstPath + ']');
}

Example 87 with IgfsPath

use of org.apache.ignite.igfs.IgfsPath in project ignite by apache.

the class IgfsMetaManager method createFileOrDirectory.

/**
 * Create file or directory.
 *
 * @param dir Directory flag.
 * @param pathIds Path IDs.
 * @param lockInfos Lock infos.
 * @param dirProps Directory properties.
 * @param fileProps File properties.
 * @param blockSize Block size.
 * @param affKey Affinity key.
 * @param evictExclude Evict exclude flag.
 * @param secondaryCtx Secondary file system create context.
 * @param secondaryOutHolder Secondary output stream holder.
 * @return Result.
 * @throws IgniteCheckedException If failed.
 */
@SuppressWarnings("unchecked")
private IgfsPathsCreateResult createFileOrDirectory(boolean dir, IgfsPathIds pathIds, Map<IgniteUuid, IgfsEntryInfo> lockInfos, Map<String, String> dirProps, Map<String, String> fileProps, int blockSize, @Nullable IgniteUuid affKey, boolean evictExclude, @Nullable IgfsSecondaryFileSystemCreateContext secondaryCtx, @Nullable T1<OutputStream> secondaryOutHolder) throws IgniteCheckedException {
    // This is our starting point.
    int lastExistingIdx = pathIds.lastExistingIndex();
    IgfsEntryInfo lastExistingInfo = lockInfos.get(pathIds.lastExistingId());
    // If current info already contains entry with the same name as it's child, then something
    // has changed concurrently. We must re-try because we cannot get info of this unexpected
    // element due to possible deadlocks.
    int curIdx = lastExistingIdx + 1;
    String curPart = pathIds.part(curIdx);
    IgniteUuid curId = pathIds.surrogateId(curIdx);
    if (lastExistingInfo.hasChild(curPart))
        return null;
    // Create entry in the secondary file system if needed.
    if (secondaryCtx != null) {
        assert secondaryOutHolder != null;
        secondaryOutHolder.set(secondaryCtx.create());
    }
    Map<IgniteUuid, EntryProcessor> procMap = new HashMap<>();
    // First step: add new entry to the last existing element.
    procMap.put(lastExistingInfo.id(), new IgfsMetaDirectoryListingAddProcessor(curPart, new IgfsListingEntry(curId, dir || !pathIds.isLastIndex(curIdx))));
    // Events support.
    IgfsPath lastCreatedPath = pathIds.lastExistingPath();
    List<IgfsPath> createdPaths = new ArrayList<>(pathIds.count() - curIdx);
    // Second step: create middle directories.
    long curTime = System.currentTimeMillis();
    while (curIdx < pathIds.count() - 1) {
        lastCreatedPath = new IgfsPath(lastCreatedPath, curPart);
        int nextIdx = curIdx + 1;
        String nextPart = pathIds.part(nextIdx);
        IgniteUuid nextId = pathIds.surrogateId(nextIdx);
        long accessTime;
        long modificationTime;
        Map<String, String> props;
        if (secondaryCtx != null) {
            accessTime = 0L;
            modificationTime = 0L;
            props = null;
        } else {
            accessTime = curTime;
            modificationTime = curTime;
            props = dirProps;
        }
        procMap.put(curId, new IgfsMetaDirectoryCreateProcessor(accessTime, modificationTime, props, nextPart, new IgfsListingEntry(nextId, dir || !pathIds.isLastIndex(nextIdx))));
        // Save event.
        createdPaths.add(lastCreatedPath);
        // Advance things further.
        curIdx++;
        curPart = nextPart;
        curId = nextId;
    }
    // Third step: create leaf.
    if (dir) {
        long accessTime;
        long modificationTime;
        Map<String, String> props;
        if (secondaryCtx != null) {
            accessTime = 0L;
            modificationTime = 0L;
            props = null;
        } else {
            accessTime = curTime;
            modificationTime = curTime;
            props = dirProps;
        }
        procMap.put(curId, new IgfsMetaDirectoryCreateProcessor(accessTime, modificationTime, props));
    } else {
        long newAccessTime;
        long newModificationTime;
        Map<String, String> newProps;
        long newLen;
        int newBlockSize;
        if (secondaryCtx != null) {
            newAccessTime = 0L;
            newModificationTime = 0L;
            newProps = null;
        } else {
            newAccessTime = curTime;
            newModificationTime = curTime;
            newProps = fileProps;
        }
        newLen = 0L;
        newBlockSize = blockSize;
        procMap.put(curId, new IgfsMetaFileCreateProcessor(newAccessTime, newModificationTime, newProps, newBlockSize, affKey, createFileLockId(false), evictExclude, newLen));
    }
    createdPaths.add(pathIds.path());
    // Execute cache operations.
    Map<Object, EntryProcessorResult> invokeRes = ((IgniteInternalCache) id2InfoPrj).invokeAll(procMap);
    IgfsEntryInfo info = (IgfsEntryInfo) invokeRes.get(curId).get();
    return new IgfsPathsCreateResult(createdPaths, info);
}

Example 88 with IgfsPath

use of org.apache.ignite.igfs.IgfsPath in project ignite by apache.

the class IgfsAbstractSelfTest method checkDeadlocks.

/**
 * Check deadlocks by creating complex directories structure and then executing chaotic operations on it. A lot of
 * exception are expected here. We are not interested in them. Instead, we want to ensure that no deadlocks occur
 * during execution.
 *
 * @param lvlCnt Total levels in folder hierarchy.
 * @param childrenDirPerLvl How many children directories to create per level.
 * @param childrenFilePerLvl How many children file to create per level.
 * @param primaryLvlCnt How many levels will exist in the primary file system before check start.
 * @param renCnt How many renames to perform.
 * @param delCnt How many deletes to perform.
 * @param updateCnt How many updates to perform.
 * @param mkdirsCnt How many directory creations to perform.
 * @param createCnt How many file creations to perform.
 * @throws Exception If failed.
 */
@SuppressWarnings("ConstantConditions")
private void checkDeadlocks(final int lvlCnt, final int childrenDirPerLvl, final int childrenFilePerLvl, int primaryLvlCnt, int renCnt, int delCnt, int updateCnt, int mkdirsCnt, int createCnt) throws Exception {
    assert childrenDirPerLvl > 0;
    // First define file system structure.
    final Map<Integer, List<IgfsPath>> dirPaths = new HashMap<>();
    final Map<Integer, List<IgfsPath>> filePaths = new HashMap<>();
    Queue<IgniteBiTuple<Integer, IgfsPath>> queue = new ArrayDeque<>();
    // Add root directory.
    queue.add(F.t(0, IgfsPath.ROOT));
    while (!queue.isEmpty()) {
        IgniteBiTuple<Integer, IgfsPath> entry = queue.poll();
        int lvl = entry.getKey();
        if (lvl < lvlCnt) {
            int newLvl = lvl + 1;
            for (int i = 0; i < childrenDirPerLvl; i++) {
                IgfsPath path = new IgfsPath(entry.getValue(), "dir-" + newLvl + "-" + i);
                queue.add(F.t(newLvl, path));
                if (!dirPaths.containsKey(newLvl))
                    dirPaths.put(newLvl, new ArrayList<IgfsPath>());
                dirPaths.get(newLvl).add(path);
            }
            for (int i = 0; i < childrenFilePerLvl; i++) {
                IgfsPath path = new IgfsPath(entry.getValue(), "file-" + newLvl + "-" + i);
                if (!filePaths.containsKey(newLvl))
                    filePaths.put(newLvl, new ArrayList<IgfsPath>());
                filePaths.get(newLvl).add(path);
            }
        }
    }
    // Now as we have all paths defined, plan operations on them.
    final Random rand = new Random(SEED);
    final int totalOpCnt = renCnt + delCnt + updateCnt + mkdirsCnt + createCnt;
    if (totalOpCnt == 0)
        throw new RuntimeException("Operations count is zero.");
    final CyclicBarrier barrier = new CyclicBarrier(totalOpCnt);
    Collection<Thread> threads = new ArrayList<>(totalOpCnt);
    // Renames.
    for (int i = 0; i < renCnt; i++) {
        Runnable r = new Runnable() {

            @Override
            public void run() {
                try {
                    int fromLvl = rand.nextInt(lvlCnt) + 1;
                    int toLvl = rand.nextInt(lvlCnt) + 1;
                    List<IgfsPath> fromPaths;
                    List<IgfsPath> toPaths;
                    if (rand.nextInt(childrenDirPerLvl + childrenFilePerLvl) < childrenDirPerLvl) {
                        // Rename directories.
                        fromPaths = dirPaths.get(fromLvl);
                        toPaths = dirPaths.get(toLvl);
                    } else {
                        // Rename files.
                        fromPaths = filePaths.get(fromLvl);
                        toPaths = filePaths.get(toLvl);
                    }
                    IgfsPath fromPath = fromPaths.get(rand.nextInt(fromPaths.size()));
                    IgfsPath toPath = toPaths.get(rand.nextInt(toPaths.size()));
                    U.awaitQuiet(barrier);
                    igfs.rename(fromPath, toPath);
                } catch (IgniteException ignore) {
                // No-op.
                }
            }
        };
        threads.add(new Thread(r));
    }
    // Deletes.
    for (int i = 0; i < delCnt; i++) {
        Runnable r = new Runnable() {

            @Override
            public void run() {
                try {
                    int lvl = rand.nextInt(lvlCnt) + 1;
                    IgfsPath path = rand.nextInt(childrenDirPerLvl + childrenFilePerLvl) < childrenDirPerLvl ? dirPaths.get(lvl).get(rand.nextInt(dirPaths.get(lvl).size())) : filePaths.get(lvl).get(rand.nextInt(filePaths.get(lvl).size()));
                    U.awaitQuiet(barrier);
                    igfs.delete(path, true);
                } catch (IgniteException ignore) {
                // No-op.
                }
            }
        };
        threads.add(new Thread(r));
    }
    // Updates.
    for (int i = 0; i < updateCnt; i++) {
        Runnable r = new Runnable() {

            @Override
            public void run() {
                try {
                    int lvl = rand.nextInt(lvlCnt) + 1;
                    IgfsPath path = rand.nextInt(childrenDirPerLvl + childrenFilePerLvl) < childrenDirPerLvl ? dirPaths.get(lvl).get(rand.nextInt(dirPaths.get(lvl).size())) : filePaths.get(lvl).get(rand.nextInt(filePaths.get(lvl).size()));
                    U.awaitQuiet(barrier);
                    igfs.update(path, properties("owner", "group", null));
                } catch (IgniteException ignore) {
                // No-op.
                }
            }
        };
        threads.add(new Thread(r));
    }
    // Directory creations.
    final AtomicInteger dirCtr = new AtomicInteger();
    for (int i = 0; i < mkdirsCnt; i++) {
        Runnable r = new Runnable() {

            @Override
            public void run() {
                try {
                    int lvl = rand.nextInt(lvlCnt) + 1;
                    IgfsPath parentPath = dirPaths.get(lvl).get(rand.nextInt(dirPaths.get(lvl).size()));
                    IgfsPath path = new IgfsPath(parentPath, "newDir-" + dirCtr.incrementAndGet());
                    U.awaitQuiet(barrier);
                    igfs.mkdirs(path);
                } catch (IgniteException ignore) {
                // No-op.
                }
            }
        };
        threads.add(new Thread(r));
    }
    // File creations.
    final AtomicInteger fileCtr = new AtomicInteger();
    for (int i = 0; i < createCnt; i++) {
        Runnable r = new Runnable() {

            @Override
            public void run() {
                try {
                    int lvl = rand.nextInt(lvlCnt) + 1;
                    IgfsPath parentPath = dirPaths.get(lvl).get(rand.nextInt(dirPaths.get(lvl).size()));
                    IgfsPath path = new IgfsPath(parentPath, "newFile-" + fileCtr.incrementAndGet());
                    U.awaitQuiet(barrier);
                    IgfsOutputStream os = null;
                    try {
                        os = igfs.create(path, true);
                        os.write(chunk);
                    } finally {
                        U.closeQuiet(os);
                    }
                } catch (IOException | IgniteException ignore) {
                // No-op.
                }
            }
        };
        threads.add(new Thread(r));
    }
    // Create file/directory structure.
    for (int i = 0; i < lvlCnt; i++) {
        int lvl = i + 1;
        boolean targetToPrimary = !dual || lvl <= primaryLvlCnt;
        IgfsPath[] dirs = dirPaths.get(lvl).toArray(new IgfsPath[dirPaths.get(lvl).size()]);
        IgfsPath[] files = filePaths.get(lvl).toArray(new IgfsPath[filePaths.get(lvl).size()]);
        if (targetToPrimary)
            create(igfs, dirs, files);
        else
            create(igfsSecondary, dirs, files);
    }
    // Start all threads and wait for them to finish.
    for (Thread thread : threads) thread.start();
    U.joinThreads(threads, null);
}

Example 89 with IgfsPath

use of org.apache.ignite.igfs.IgfsPath in project ignite by apache.

the class IgfsAbstractSelfTest method testMoveFile.

/**
 * Test move in case both local and remote file systems have the same folder structure and the path being renamed is
 * a file.
 *
 * @throws Exception If failed.
 */
public void testMoveFile() throws Exception {
    create(igfs, paths(DIR, SUBDIR, DIR_NEW, SUBDIR_NEW), paths(FILE));
    igfs.rename(FILE, SUBDIR_NEW);
    checkExist(igfs, igfsSecondary, new IgfsPath(SUBDIR_NEW, FILE.name()));
    checkNotExist(igfs, igfsSecondary, FILE);
}

Example 90 with IgfsPath

use of org.apache.ignite.igfs.IgfsPath in project ignite by apache.

the class IgfsAbstractSelfTest method testMkdirs.

/**
 * Test mkdirs in case both local and remote file systems have the same folder structure.
 *
 * @throws Exception If failed.
 */
@SuppressWarnings("ConstantConditions")
public void testMkdirs() throws Exception {
    if (!propertiesSupported())
        return;
    // mkdirs command doesn't propagate user info.
    Map<String, String> props = properties(null, null, "0555");
    igfs.mkdirs(new IgfsPath("/x"), null);
    checkExist(igfs, igfsSecondary, new IgfsPath("/x"));
    igfs.mkdirs(new IgfsPath("/k/l"), null);
    checkExist(igfs, igfsSecondary, new IgfsPath("/k/l"));
    igfs.mkdirs(new IgfsPath("/x/y"), null);
    checkExist(igfs, igfsSecondary, new IgfsPath("/x/y"));
    igfs.mkdirs(new IgfsPath("/a/b/c/d"), null);
    checkExist(igfs, igfsSecondary, new IgfsPath("/a/b/c/d"));
    igfs.mkdirs(new IgfsPath("/a/b/c/d/e"), null);
    checkExist(igfs, igfsSecondary, new IgfsPath("/a/b/c/d/e"));
    // "f" is a file.
    create(igfs, null, new IgfsPath[] { new IgfsPath("/d/f") });
    checkExist(igfs, igfsSecondary, new IgfsPath("/d/f"));
    assertTrue(igfs.info(new IgfsPath("/d/f")).isFile());
    try {
        igfs.mkdirs(new IgfsPath("/d/f"), null);
        fail("IgfsParentNotDirectoryException expected.");
    } catch (IgfsParentNotDirectoryException ignore) {
    // No-op.
    } catch (IgfsException e) {
        // Currently Ok for Hadoop fs:
        if (!getClass().getSimpleName().startsWith("Hadoop"))
            throw e;
    }
    try {
        igfs.mkdirs(new IgfsPath("/d/f/something/else"), null);
        fail("IgfsParentNotDirectoryException expected.");
    } catch (IgfsParentNotDirectoryException ignore) {
    // No-op.
    } catch (IgfsException e) {
        // Currently Ok for Hadoop fs:
        if (!getClass().getSimpleName().startsWith("Hadoop"))
            throw e;
    }
    create(igfs, paths(DIR, SUBDIR), null);
    igfs.mkdirs(SUBSUBDIR, props);
    // Ensure that directory was created and properties are propagated.
    checkExist(igfs, igfsSecondary, SUBSUBDIR);
    if (permissionsSupported()) {
        if (dual)
            // Check only permissions because user and group will always be present in Hadoop Fs.
            assertEquals(props.get(IgfsUtils.PROP_PERMISSION), igfsSecondary.permissions(SUBSUBDIR.toString()));
        // We check only permission because IGFS client adds username and group name explicitly.
        assertEquals(props.get(IgfsUtils.PROP_PERMISSION), igfs.info(SUBSUBDIR).properties().get(IgfsUtils.PROP_PERMISSION));
    }
}