Examples with IgfsBlockLocation org.apache.ignite.igfs.IgfsBlockLocation used on opensource projects

Example 11 with IgfsBlockLocation

use of org.apache.ignite.igfs.IgfsBlockLocation in project ignite by apache.

the class IgfsDataManagerSelfTest method checkAffinity.

/**
 * Checks affinity validity.
 *
 * @param blockSize Block size.
 * @param info File info.
 * @param affinity Affinity block locations to check.
 */
private void checkAffinity(int blockSize, IgfsEntryInfo info, Iterable<IgfsBlockLocation> affinity) {
    for (IgfsBlockLocation loc : affinity) {
        info("Going to check IGFS block location: " + loc);
        int block = (int) (loc.start() / blockSize);
        int endPos;
        do {
            IgfsBlockKey key = new IgfsBlockKey(info.id(), info.fileMap().affinityKey(block * blockSize, false), false, block);
            ClusterNode affNode = grid(0).affinity(grid(0).igfsx("igfs").configuration().getDataCacheConfiguration().getName()).mapKeyToNode(key);
            assertTrue("Failed to find node in affinity [dataMgr=" + loc.nodeIds() + ", nodeId=" + affNode.id() + ", block=" + block + ']', loc.nodeIds().contains(affNode.id()));
            endPos = (block + 1) * blockSize;
            block++;
        } while (endPos < loc.start() + loc.length());
    }
}

Example 12 with IgfsBlockLocation

use of org.apache.ignite.igfs.IgfsBlockLocation in project ignite by apache.

the class IgniteHadoopFileSystemAbstractSelfTest method testZeroReplicationFactor.

/**
 * @throws Exception If failed.
 */
public void testZeroReplicationFactor() throws Exception {
    // This test doesn't make sense for any mode except of PRIMARY.
    if (mode == PRIMARY) {
        Path igfsHome = new Path(PRIMARY_URI);
        Path file = new Path(igfsHome, "someFile");
        try (FSDataOutputStream out = fs.create(file, (short) 0)) {
            out.write(new byte[1024 * 1024]);
        }
        IgniteFileSystem igfs = grid(0).fileSystem("igfs");
        IgfsPath filePath = new IgfsPath("/someFile");
        IgfsFile fileInfo = igfs.info(filePath);
        awaitPartitionMapExchange();
        Collection<IgfsBlockLocation> locations = igfs.affinity(filePath, 0, fileInfo.length());
        assertEquals(1, locations.size());
        IgfsBlockLocation location = F.first(locations);
        assertEquals(1, location.nodeIds().size());
    }
}

Example 13 with IgfsBlockLocation

use of org.apache.ignite.igfs.IgfsBlockLocation in project ignite by apache.

the class Hadoop1OverIgfsProxyTest method testAffinity.

/**
 * @throws Exception If failed.
 */
public void testAffinity() throws Exception {
    long fileSize = 32 * 1024 * 1024;
    IgfsPath filePath = new IgfsPath("/file");
    try (OutputStream os = igfs.create(filePath, true)) {
        for (int i = 0; i < fileSize / chunk.length; ++i) os.write(chunk);
    }
    long len = igfs.info(filePath).length();
    int start = 0;
    // Check default maxLen (maxLen = 0)
    for (int i = 0; i < igfs.context().data().groupBlockSize() / 1024; i++) {
        Collection<IgfsBlockLocation> blocks = igfs.affinity(filePath, start, len);
        assertEquals(F.first(blocks).start(), start);
        assertEquals(start + len, F.last(blocks).start() + F.last(blocks).length());
        len -= 1024 * 2;
        start += 1024;
    }
}

Example 14 with IgfsBlockLocation

use of org.apache.ignite.igfs.IgfsBlockLocation in project ignite by apache.

the class IgfsStreamsSelfTest method testCreateFileColocated.

/**
 * @throws Exception If failed.
 */
public void testCreateFileColocated() throws Exception {
    IgfsPath path = new IgfsPath("/colocated");
    UUID uuid = UUID.randomUUID();
    IgniteUuid affKey;
    long idx = 0;
    while (true) {
        affKey = new IgniteUuid(uuid, idx);
        if (grid(0).affinity(grid(0).igfsx("igfs").configuration().getDataCacheConfiguration().getName()).mapKeyToNode(affKey).id().equals(grid(0).localNode().id()))
            break;
        idx++;
    }
    try (IgfsOutputStream out = fs.create(path, 1024, true, affKey, 0, 1024, null)) {
        // Write 5M, should be enough to test distribution.
        for (int i = 0; i < 15; i++) out.write(new byte[1024 * 1024]);
    }
    IgfsFile info = fs.info(path);
    Collection<IgfsBlockLocation> affNodes = fs.affinity(path, 0, info.length());
    assertEquals(1, affNodes.size());
    Collection<UUID> nodeIds = F.first(affNodes).nodeIds();
    assertEquals(1, nodeIds.size());
    assertEquals(grid(0).localNode().id(), F.first(nodeIds));
}

Example 15 with IgfsBlockLocation

use of org.apache.ignite.igfs.IgfsBlockLocation in project ignite by apache.

the class IgfsStreamsSelfTest method testCreateFile.

/**
 * Test file creation.
 *
 * @param path Path to file to store.
 * @param size Size of file to store.
 * @param salt Salt for file content generation.
 * @throws Exception In case of any exception.
 */
private void testCreateFile(final IgfsPath path, final long size, final int salt) throws Exception {
    info("Create file [path=" + path + ", size=" + size + ", salt=" + salt + ']');
    final AtomicInteger cnt = new AtomicInteger(0);
    final Collection<IgfsPath> cleanUp = new ConcurrentLinkedQueue<>();
    long time = runMultiThreaded(new Callable<Object>() {

        @Override
        public Object call() throws Exception {
            int id = cnt.incrementAndGet();
            IgfsPath f = new IgfsPath(path.parent(), "asdf" + (id > 1 ? "-" + id : ""));
            try (IgfsOutputStream out = fs.create(f, 0, true, null, 0, 1024, null)) {
                assertNotNull(out);
                // Add all created into cleanup list.
                cleanUp.add(f);
                U.copy(new IgfsTestInputStream(size, salt), out);
            }
            return null;
        }
    }, WRITING_THREADS_CNT, "perform-multi-thread-writing");
    if (time > 0) {
        double rate = size * 1000. / time / 1024 / 1024;
        info(String.format("Write file [path=%s, size=%d kB, rate=%2.1f MB/s]", path, WRITING_THREADS_CNT * size / 1024, WRITING_THREADS_CNT * rate));
    }
    info("Read and validate saved file: " + path);
    final InputStream expIn = new IgfsTestInputStream(size, salt);
    final IgfsInputStream actIn = fs.open(path, CFG_BLOCK_SIZE * READING_THREADS_CNT * 11 / 10);
    // Validate continuous reading of whole file.
    assertEqualStreams(expIn, actIn, size, null);
    // Validate random seek and reading.
    final Random rnd = new Random();
    runMultiThreaded(new Callable<Object>() {

        @Override
        public Object call() throws Exception {
            long skip = Math.abs(rnd.nextLong() % (size + 1));
            long range = Math.min(size - skip, rnd.nextInt(CFG_BLOCK_SIZE * 400));
            assertEqualStreams(new IgfsTestInputStream(size, salt), actIn, range, skip);
            return null;
        }
    }, READING_THREADS_CNT, "validate-multi-thread-reading");
    expIn.close();
    actIn.close();
    info("Get stored file info: " + path);
    IgfsFile desc = fs.info(path);
    info("Validate stored file info: " + desc);
    assertNotNull(desc);
    if (log.isDebugEnabled())
        log.debug("File descriptor: " + desc);
    Collection<IgfsBlockLocation> aff = fs.affinity(path, 0, desc.length());
    assertFalse("Affinity: " + aff, desc.length() != 0 && aff.isEmpty());
    int blockSize = desc.blockSize();
    assertEquals("File size", size, desc.length());
    assertEquals("Binary block size", CFG_BLOCK_SIZE, blockSize);
    // assertEquals("Permission", "rwxr-xr-x", desc.getPermission().toString());
    // assertEquals("Permission sticky bit marks this is file", false, desc.getPermission().getStickyBit());
    assertEquals("Type", true, desc.isFile());
    assertEquals("Type", false, desc.isDirectory());
    info("Cleanup files: " + cleanUp);
    for (IgfsPath f : cleanUp) {
        fs.delete(f, true);
        assertNull(fs.info(f));
    }
}