Examples with IgfsPath org.apache.ignite.igfs.IgfsPath used on opensource projects

Example 66 with IgfsPath

use of org.apache.ignite.igfs.IgfsPath in project ignite by apache.

the class IgniteHadoopWeightedMapReducePlanner method igfsAffinityNodesForSplit.

/**
 * Get IGFS affinity nodes for split if possible.
 * <p>
 * Order in the returned collection *is* significant, meaning that nodes containing more data
 * go first. This way, the 1st nodes in the collection considered to be preferable for scheduling.
 *
 * @param split Input split.
 * @return IGFS affinity or {@code null} if IGFS is not available.
 * @throws IgniteCheckedException If failed.
 */
@Nullable
private Collection<UUID> igfsAffinityNodesForSplit(HadoopInputSplit split) throws IgniteCheckedException {
    if (split instanceof HadoopFileBlock) {
        HadoopFileBlock split0 = (HadoopFileBlock) split;
        if (IgniteFileSystem.IGFS_SCHEME.equalsIgnoreCase(split0.file().getScheme())) {
            HadoopIgfsEndpoint endpoint = new HadoopIgfsEndpoint(split0.file().getAuthority());
            IgfsEx igfs = (IgfsEx) ((IgniteEx) ignite).igfsx(endpoint.igfs());
            if (igfs != null && !igfs.isProxy(split0.file())) {
                IgfsPath path = new IgfsPath(split0.file());
                if (igfs.exists(path)) {
                    Collection<IgfsBlockLocation> blocks;
                    try {
                        blocks = igfs.affinity(path, split0.start(), split0.length());
                    } catch (IgniteException e) {
                        throw new IgniteCheckedException("Failed to get IGFS file block affinity [path=" + path + ", start=" + split0.start() + ", len=" + split0.length() + ']', e);
                    }
                    assert blocks != null;
                    if (blocks.size() == 1)
                        return blocks.iterator().next().nodeIds();
                    else {
                        // The most "local" nodes go first.
                        Map<UUID, Long> idToLen = new HashMap<>();
                        for (IgfsBlockLocation block : blocks) {
                            for (UUID id : block.nodeIds()) {
                                Long len = idToLen.get(id);
                                idToLen.put(id, len == null ? block.length() : block.length() + len);
                            }
                        }
                        // Sort the nodes in non-ascending order by contained data lengths.
                        Map<NodeIdAndLength, UUID> res = new TreeMap<>();
                        for (Map.Entry<UUID, Long> idToLenEntry : idToLen.entrySet()) {
                            UUID id = idToLenEntry.getKey();
                            res.put(new NodeIdAndLength(id, idToLenEntry.getValue()), id);
                        }
                        return new LinkedHashSet<>(res.values());
                    }
                }
            }
        }
    }
    return null;
}

Example 67 with IgfsPath

use of org.apache.ignite.igfs.IgfsPath in project ignite by apache.

the class HadoopIgfsSecondaryFileSystemDelegateImpl method info.

/**
 * {@inheritDoc}
 */
@Override
public IgfsFile info(final IgfsPath path) {
    try {
        final FileStatus status = fileSystemForUser().getFileStatus(convert(path));
        if (status == null)
            return null;
        final Map<String, String> props = properties(status);
        return new IgfsFileImpl(new IgfsFile() {

            @Override
            public IgfsPath path() {
                return path;
            }

            @Override
            public boolean isFile() {
                return status.isFile();
            }

            @Override
            public boolean isDirectory() {
                return status.isDirectory();
            }

            @Override
            public int blockSize() {
                // By convention directory has blockSize == 0, while file has blockSize > 0:
                return isDirectory() ? 0 : (int) status.getBlockSize();
            }

            @Override
            public long groupBlockSize() {
                return status.getBlockSize();
            }

            @Override
            public long accessTime() {
                return status.getAccessTime();
            }

            @Override
            public long modificationTime() {
                return status.getModificationTime();
            }

            @Override
            public String property(String name) throws IllegalArgumentException {
                String val = props.get(name);
                if (val == null)
                    throw new IllegalArgumentException("File property not found [path=" + path + ", name=" + name + ']');
                return val;
            }

            @Nullable
            @Override
            public String property(String name, @Nullable String dfltVal) {
                String val = props.get(name);
                return val == null ? dfltVal : val;
            }

            @Override
            public long length() {
                return status.getLen();
            }

            /**
             * {@inheritDoc}
             */
            @Override
            public Map<String, String> properties() {
                return props;
            }
        }, 0);
    } catch (FileNotFoundException ignore) {
        return null;
    } catch (IOException e) {
        throw handleSecondaryFsError(e, "Failed to get file status [path=" + path + "]");
    }
}

Example 68 with IgfsPath

use of org.apache.ignite.igfs.IgfsPath in project ignite by apache.

the class HadoopTaskExecutionSelfTest method prepareFile.

/**
 * @param fileName File name.
 * @param lineCnt Line count.
 * @throws Exception If failed.
 */
private void prepareFile(String fileName, int lineCnt) throws Exception {
    IgniteFileSystem igfs = grid(0).fileSystem(igfsName);
    try (OutputStream os = igfs.create(new IgfsPath(fileName), true)) {
        PrintWriter w = new PrintWriter(new OutputStreamWriter(os));
        for (int i = 0; i < lineCnt; i++) w.print("Hello, Hadoop map-reduce!\n");
        w.flush();
    }
}

Example 69 with IgfsPath

use of org.apache.ignite.igfs.IgfsPath in project ignite by apache.

the class HadoopTasksAllVersionsTest method testAllTasks.

/**
 * Tests all job in complex.
 * Runs 2 chains of map-combine tasks and sends result into one reduce task.
 *
 * @throws Exception If fails.
 */
@SuppressWarnings("ConstantConditions")
public void testAllTasks() throws Exception {
    IgfsPath inDir = new IgfsPath(PATH_INPUT);
    igfs.mkdirs(inDir);
    IgfsPath inFile = new IgfsPath(inDir, HadoopWordCount2.class.getSimpleName() + "-input");
    URI inFileUri = URI.create(igfsScheme() + inFile.toString());
    generateTestFile(inFile.toString(), "red", 100, "blue", 200, "green", 150, "yellow", 70);
    // Split file into two blocks
    long fileLen = igfs.info(inFile).length();
    Long l = fileLen / 2;
    HadoopFileBlock fileBlock1 = new HadoopFileBlock(HOSTS, inFileUri, 0, l);
    HadoopFileBlock fileBlock2 = new HadoopFileBlock(HOSTS, inFileUri, l, fileLen - l);
    HadoopJobEx gridJob = getHadoopJob(inFileUri.toString(), igfsScheme() + PATH_OUTPUT);
    HadoopTestTaskContext combine1Ctx = runMapCombineTask(fileBlock1, gridJob);
    HadoopTestTaskContext combine2Ctx = runMapCombineTask(fileBlock2, gridJob);
    // Prepare input for combine
    HadoopTaskInfo taskInfo = new HadoopTaskInfo(HadoopTaskType.REDUCE, gridJob.id(), 0, 0, null);
    HadoopTestTaskContext reduceCtx = new HadoopTestTaskContext(taskInfo, gridJob);
    reduceCtx.makeTreeOfWritables(combine1Ctx.mockOutput());
    reduceCtx.makeTreeOfWritables(combine2Ctx.mockOutput());
    reduceCtx.run();
    reduceCtx.taskInfo(new HadoopTaskInfo(HadoopTaskType.COMMIT, gridJob.id(), 0, 0, null));
    reduceCtx.run();
    assertEquals("blue\t200\n" + "green\t150\n" + "red\t100\n" + "yellow\t70\n", readAndSortFile(PATH_OUTPUT + "/" + getOutputFileNamePrefix() + "00000"));
}

Example 70 with IgfsPath

use of org.apache.ignite.igfs.IgfsPath in project ignite by apache.

the class IgfsImpl method await.

/**
 * {@inheritDoc}
 */
@Override
public void await(IgfsPath... paths) {
    assert paths != null;
    for (Map.Entry<IgfsPath, IgfsFileWorkerBatch> workerEntry : workerMap.entrySet()) {
        IgfsPath workerPath = workerEntry.getKey();
        boolean await = false;
        for (IgfsPath path : paths) {
            if (workerPath.isSubDirectoryOf(path) || F.eq(workerPath, path)) {
                await = true;
                break;
            }
        }
        if (await) {
            IgfsFileWorkerBatch batch = workerEntry.getValue();
            if (batch != null) {
                try {
                    // batches on current path and all it's known children.
                    if (batch.finishing())
                        batch.await();
                } catch (IgniteCheckedException ignore) {
                // No-op.
                }
            }
        }
    }
}