Examples with IgfsFile org.apache.ignite.igfs.IgfsFile used on opensource projects

Example 1 with IgfsFile

use of org.apache.ignite.igfs.IgfsFile in project ignite by apache.

the class IgfsTask method map.

/**
 * {@inheritDoc}
 */
@Nullable
@Override
public final Map<? extends ComputeJob, ClusterNode> map(List<ClusterNode> subgrid, @Nullable IgfsTaskArgs<T> args) {
    assert ignite != null;
    assert args != null;
    IgniteFileSystem fs = ignite.fileSystem(args.igfsName());
    IgfsProcessorAdapter igfsProc = ((IgniteKernal) ignite).context().igfs();
    Map<ComputeJob, ClusterNode> splitMap = new HashMap<>();
    Map<UUID, ClusterNode> nodes = mapSubgrid(subgrid);
    for (IgfsPath path : args.paths()) {
        IgfsFile file = fs.info(path);
        if (file == null) {
            if (args.skipNonExistentFiles())
                continue;
            else
                throw new IgniteException("Failed to process IGFS file because it doesn't exist: " + path);
        }
        Collection<IgfsBlockLocation> aff = fs.affinity(path, 0, file.length(), args.maxRangeLength());
        long totalLen = 0;
        for (IgfsBlockLocation loc : aff) {
            ClusterNode node = null;
            for (UUID nodeId : loc.nodeIds()) {
                node = nodes.get(nodeId);
                if (node != null)
                    break;
            }
            if (node == null)
                throw new IgniteException("Failed to find any of block affinity nodes in subgrid [loc=" + loc + ", subgrid=" + subgrid + ']');
            IgfsJob job = createJob(path, new IgfsFileRange(file.path(), loc.start(), loc.length()), args);
            if (job != null) {
                ComputeJob jobImpl = igfsProc.createJob(job, fs.name(), file.path(), loc.start(), loc.length(), args.recordResolver());
                splitMap.put(jobImpl, node);
            }
            totalLen += loc.length();
        }
        assert totalLen == file.length();
    }
    return splitMap;
}

Example 2 with IgfsFile

use of org.apache.ignite.igfs.IgfsFile in project ignite by apache.

the class IgfsAbstractSelfTest method testInfo.

/**
 * Test info routine when the path exists both locally and remotely.
 *
 * @throws Exception If failed.
 */
public void testInfo() throws Exception {
    create(igfs, paths(DIR), null);
    IgfsFile info = igfs.info(DIR);
    assert info != null;
    assertEquals(DIR, info.path());
}

Example 3 with IgfsFile

use of org.apache.ignite.igfs.IgfsFile in project ignite by apache.

the class IgfsDualAbstractSelfTest method checkParentListingTime.

/**
 * Checks file access & modification time equality in the file itself and in the same file found through
 * the listing of its parent.
 *
 * @param fs The file system.
 * @param p The file path.
 *
 * @return Tuple of access and modification times of the file.
 */
private T2<Long, Long> checkParentListingTime(IgfsSecondaryFileSystem fs, IgfsPath p) {
    IgfsFile f0 = fs.info(p);
    T2<Long, Long> t0 = new T2<>(f0.accessTime(), f0.modificationTime());
    // Root cannot be seen through the parent listing:
    if (!F.eq(IgfsPath.ROOT, p)) {
        assertNotNull(f0);
        Collection<IgfsFile> listing = fs.listFiles(p.parent());
        IgfsFile f1 = null;
        for (IgfsFile fi : listing) {
            if (F.eq(fi.path(), p)) {
                f1 = fi;
                break;
            }
        }
        // file should be found in parent listing.
        assertNotNull(f1);
        T2<Long, Long> t1 = new T2<>(f1.accessTime(), f1.modificationTime());
        assertEquals(t0, t1);
    }
    return t0;
}

Example 4 with IgfsFile

use of org.apache.ignite.igfs.IgfsFile in project ignite by apache.

the class HadoopIgfs20FileSystemAbstractSelfTest method testZeroReplicationFactor.

/**
 * @throws Exception If failed.
 */
public void testZeroReplicationFactor() throws Exception {
    // This test doesn't make sense for any mode except of PRIMARY.
    if (mode == PRIMARY) {
        Path igfsHome = new Path(primaryFsUri);
        Path file = new Path(igfsHome, "someFile");
        try (FSDataOutputStream out = fs.create(file, EnumSet.noneOf(CreateFlag.class), Options.CreateOpts.perms(FsPermission.getDefault()), Options.CreateOpts.repFac((short) 1))) {
            out.write(new byte[1024 * 1024]);
        }
        IgniteFileSystem igfs = grid(0).fileSystem("igfs");
        IgfsPath filePath = new IgfsPath("/someFile");
        IgfsFile fileInfo = igfs.info(filePath);
        Collection<IgfsBlockLocation> locations = igfs.affinity(filePath, 0, fileInfo.length());
        assertEquals(1, locations.size());
        IgfsBlockLocation location = F.first(locations);
        assertEquals(1, location.nodeIds().size());
    }
}

Example 5 with IgfsFile

use of org.apache.ignite.igfs.IgfsFile in project ignite by apache.

the class HadoopClientProtocolSelfTest method dumpIgfs.

/**
 * Dump IGFS content.
 *
 * @param igfs IGFS.
 * @param path Path.
 * @throws Exception If failed.
 */
@SuppressWarnings("ConstantConditions")
private static void dumpIgfs(IgniteFileSystem igfs, IgfsPath path) throws Exception {
    IgfsFile file = igfs.info(path);
    assert file != null;
    System.out.println(file.path());
    if (file.isDirectory()) {
        for (IgfsPath child : igfs.listPaths(path)) dumpIgfs(igfs, child);
    } else {
        try (BufferedReader br = new BufferedReader(new InputStreamReader(igfs.open(path)))) {
            String line = br.readLine();
            while (line != null) {
                System.out.println(line);
                line = br.readLine();
            }
        }
    }
}