Examples with IgfsBlockLocation org.apache.ignite.igfs.IgfsBlockLocation used on opensource projects

Example 1 with IgfsBlockLocation

use of org.apache.ignite.igfs.IgfsBlockLocation in project ignite by apache.

the class IgfsTask method map.

/**
 * {@inheritDoc}
 */
@Nullable
@Override
public final Map<? extends ComputeJob, ClusterNode> map(List<ClusterNode> subgrid, @Nullable IgfsTaskArgs<T> args) {
    assert ignite != null;
    assert args != null;
    IgniteFileSystem fs = ignite.fileSystem(args.igfsName());
    IgfsProcessorAdapter igfsProc = ((IgniteKernal) ignite).context().igfs();
    Map<ComputeJob, ClusterNode> splitMap = new HashMap<>();
    Map<UUID, ClusterNode> nodes = mapSubgrid(subgrid);
    for (IgfsPath path : args.paths()) {
        IgfsFile file = fs.info(path);
        if (file == null) {
            if (args.skipNonExistentFiles())
                continue;
            else
                throw new IgniteException("Failed to process IGFS file because it doesn't exist: " + path);
        }
        Collection<IgfsBlockLocation> aff = fs.affinity(path, 0, file.length(), args.maxRangeLength());
        long totalLen = 0;
        for (IgfsBlockLocation loc : aff) {
            ClusterNode node = null;
            for (UUID nodeId : loc.nodeIds()) {
                node = nodes.get(nodeId);
                if (node != null)
                    break;
            }
            if (node == null)
                throw new IgniteException("Failed to find any of block affinity nodes in subgrid [loc=" + loc + ", subgrid=" + subgrid + ']');
            IgfsJob job = createJob(path, new IgfsFileRange(file.path(), loc.start(), loc.length()), args);
            if (job != null) {
                ComputeJob jobImpl = igfsProc.createJob(job, fs.name(), file.path(), loc.start(), loc.length(), args.recordResolver());
                splitMap.put(jobImpl, node);
            }
            totalLen += loc.length();
        }
        assert totalLen == file.length();
    }
    return splitMap;
}

Example 2 with IgfsBlockLocation

use of org.apache.ignite.igfs.IgfsBlockLocation in project ignite by apache.

the class IgfsDataManagerSelfTest method testAffinity2.

/**
 * @throws Exception If failed.
 */
public void testAffinity2() throws Exception {
    int blockSize = BLOCK_SIZE;
    long t = System.currentTimeMillis();
    IgfsEntryInfo info = IgfsUtils.createFile(IgniteUuid.randomUuid(), blockSize, 1024 * 1024, null, null, false, null, t, t);
    Collection<IgfsBlockLocation> affinity = mgr.affinity(info, 0, info.length());
    for (IgfsBlockLocation loc : affinity) {
        info("Going to check IGFS block location: " + loc);
        int block = (int) (loc.start() / blockSize);
        int endPos;
        do {
            IgfsBlockKey key = new IgfsBlockKey(info.id(), null, false, block);
            ClusterNode affNode = grid(0).affinity(grid(0).igfsx("igfs").configuration().getDataCacheConfiguration().getName()).mapKeyToNode(key);
            assertTrue("Failed to find node in affinity [dataMgr=" + loc.nodeIds() + ", nodeId=" + affNode.id() + ", block=" + block + ']', loc.nodeIds().contains(affNode.id()));
            endPos = (block + 1) * blockSize;
            block++;
        } while (endPos < loc.start() + loc.length());
    }
}

Example 3 with IgfsBlockLocation

use of org.apache.ignite.igfs.IgfsBlockLocation in project ignite by apache.

the class IgfsDataManagerSelfTest method testAffinity.

/**
 * Test affinity.
 *
 * @throws Exception If failed.
 */
public void testAffinity() throws Exception {
    final int blockSize = 10;
    final int grpSize = blockSize * DATA_BLOCK_GROUP_CNT;
    long t = System.currentTimeMillis();
    // IgfsFileInfo info = new IgfsFileInfo(blockSize, 0);
    IgfsEntryInfo info = IgfsUtils.createFile(IgniteUuid.randomUuid(), blockSize, 1024 * 1024, null, null, false, null, t, t);
    for (int pos = 0; pos < 5 * grpSize; pos++) {
        assertEquals("Expects no affinity for zero length.", Collections.<IgfsBlockLocation>emptyList(), mgr.affinity(info, pos, 0));
        // And no guaranties for blocks out of the group.
        for (int len = 1, maxLen = grpSize - pos % grpSize; len < maxLen; len++) {
            Collection<IgfsBlockLocation> aff = mgr.affinity(info, pos, len);
            assertEquals("Unexpected affinity: " + aff, 1, aff.size());
            IgfsBlockLocation loc = F.first(aff);
            assertEquals("Unexpected block location: " + loc, pos, loc.start());
            assertEquals("Unexpected block location: " + loc, len, loc.length());
        }
        // Validate ranges.
        for (int len = grpSize * 4 + 1, maxLen = 5 * grpSize - pos % grpSize; len < maxLen; len++) {
            Collection<IgfsBlockLocation> aff = mgr.affinity(info, pos, len);
            assertTrue("Unexpected affinity [aff=" + aff + ", pos=" + pos + ", len=" + len + ']', aff.size() <= 5);
            IgfsBlockLocation first = F.first(aff);
            assertEquals("Unexpected the first block location [aff=" + aff + ", pos=" + pos + ", len=" + len + ']', pos, first.start());
            assertTrue("Unexpected the first block location [aff=" + aff + ", pos=" + pos + ", len=" + len + ']', first.length() >= grpSize - pos % grpSize);
            IgfsBlockLocation last = F.last(aff);
            assertTrue("Unexpected the last block location [aff=" + aff + ", pos=" + pos + ", len=" + len + ']', last.start() <= (pos / grpSize + 4) * grpSize);
            assertTrue("Unexpected the last block location [aff=" + aff + ", pos=" + pos + ", len=" + len + ']', last.length() >= (pos + len - 1) % grpSize + 1);
        }
    }
}

Example 4 with IgfsBlockLocation

use of org.apache.ignite.igfs.IgfsBlockLocation in project ignite by apache.

the class IgfsDataManagerSelfTest method testAffinityFileMap.

/**
 * @throws Exception If failed.
 */
public void testAffinityFileMap() throws Exception {
    int blockSize = BLOCK_SIZE;
    long t = System.currentTimeMillis();
    IgfsEntryInfo info = IgfsUtils.createFile(IgniteUuid.randomUuid(), blockSize, 1024 * 1024, null, null, false, null, t, t);
    IgniteUuid affKey = IgniteUuid.randomUuid();
    IgfsFileMap map = new IgfsFileMap();
    map.addRange(new IgfsFileAffinityRange(3 * BLOCK_SIZE, 5 * BLOCK_SIZE - 1, affKey));
    map.addRange(new IgfsFileAffinityRange(13 * BLOCK_SIZE, 17 * BLOCK_SIZE - 1, affKey));
    info = info.fileMap(map);
    Collection<IgfsBlockLocation> affinity = mgr.affinity(info, 0, info.length());
    checkAffinity(blockSize, info, affinity);
    // Check from middle of range.
    affinity = mgr.affinity(info, 3 * BLOCK_SIZE + BLOCK_SIZE / 2, info.length());
    checkAffinity(blockSize, info, affinity);
    // Check from middle of last range.
    affinity = mgr.affinity(info, 14 * BLOCK_SIZE, info.length());
    checkAffinity(blockSize, info, affinity);
    // Check inside one range.
    affinity = mgr.affinity(info, 14 * BLOCK_SIZE, 2 * BLOCK_SIZE);
    checkAffinity(blockSize, info, affinity);
    // Check outside last range.
    affinity = mgr.affinity(info, 18 * BLOCK_SIZE, info.length());
    checkAffinity(blockSize, info, affinity);
}

Example 5 with IgfsBlockLocation

use of org.apache.ignite.igfs.IgfsBlockLocation in project ignite by apache.

the class IgfsLocalSecondaryFileSystemProxySelfTest method testAffinityMaxLen.

/**
 * @throws Exception If failed.
 */
public void testAffinityMaxLen() throws Exception {
    awaitPartitionMapExchange();
    long fileSize = 32 * 1024 * 1024;
    IgfsPath filePath = new IgfsPath("/file");
    try (OutputStream os = igfs.create(filePath, true)) {
        for (int i = 0; i < fileSize / chunk.length; ++i) os.write(chunk);
    }
    Collection<IgfsBlockLocation> blocks;
    long len = igfs.info(filePath).length();
    int start = 0;
    // Check default maxLen (maxLen = 0)
    for (int i = 0; i < igfs.context().data().groupBlockSize() / 1024; i++) {
        Collection<IgfsBlockLocation> blocks0 = igfs.affinity(filePath, start, len, 0);
        blocks = igfs.affinity(filePath, start, len, Long.MAX_VALUE);
        assertTrue(blocks0.size() > 1);
        assertEquals(blocks0.size(), blocks.size());
        assertEquals(F.first(blocks).start(), start);
        assertEquals(start + len, F.last(blocks).start() + F.last(blocks).length());
        assertEquals(blocks0, blocks);
        len -= 1024 * 2;
        start += 1024;
    }
    len = igfs.info(filePath).length();
    start = 0;
    long maxLen = igfs.context().data().groupBlockSize() * 2;
    // Different cases of start, len and maxLen
    for (int i = 0; i < igfs.context().data().groupBlockSize() / 1024; i++) {
        blocks = igfs.affinity(filePath, start, len, maxLen);
        assertEquals(F.first(blocks).start(), start);
        assertEquals(start + len, F.last(blocks).start() + F.last(blocks).length());
        long totalLen = 0;
        for (IgfsBlockLocation block : blocks) {
            totalLen += block.length();
            assert block.length() <= maxLen : "block.length() <= maxLen. [block.length=" + block.length() + ", maxLen=" + maxLen + ']';
            assert block.length() + block.start() <= start + len : "block.length() + block.start() < start + len. [block.length=" + block.length() + ", block.start()=" + block.start() + ", start=" + start + ", len=" + len + ']';
            for (IgfsBlockLocation block0 : blocks) if (!block0.equals(block))
                assert block.start() < block0.start() && block.start() + block.length() <= block0.start() || block.start() > block0.start() && block0.start() + block0.length() <= block.start() : "Blocks cross each other: block0=" + block + ", block1= " + block0;
        }
        assert totalLen == len : "Summary length of blocks must be: " + len + " actual: " + totalLen;
        len -= 1024 * 2;
        start += 1024;
        maxLen -= igfs.context().data().groupBlockSize() * 2 / 1024;
    }
}