Examples with IgfsPath org.apache.ignite.igfs.IgfsPath used on opensource projects

Example 91 with IgfsPath

use of org.apache.ignite.igfs.IgfsPath in project ignite by apache.

the class IgfsAbstractSelfTest method testSimpleWrite.

/**
 * Checks simple write.
 *
 * @throws Exception On error.
 */
public void testSimpleWrite() throws Exception {
    IgfsPath path = new IgfsPath("/file1");
    IgfsOutputStream os = igfs.create(path, 128, true, /*overwrite*/
    null, 0, 256, null);
    os.write(chunk);
    os.close();
    assert igfs.exists(path);
    checkFileContent(igfs, path, chunk);
    os = igfs.create(path, 128, true, /*overwrite*/
    null, 0, 256, null);
    assert igfs.exists(path);
    os.write(chunk);
    assert igfs.exists(path);
    os.write(chunk);
    assert igfs.exists(path);
    os.close();
    assert igfs.exists(path);
    checkFileContent(igfs, path, chunk, chunk);
}

Example 92 with IgfsPath

use of org.apache.ignite.igfs.IgfsPath in project ignite by apache.

the class IgfsAbstractSelfTest method testAppendConsistency.

/**
 * Ensure consistency of data during appending to a file.
 *
 * @throws Exception If failed.
 */
public void testAppendConsistency() throws Exception {
    if (appendSupported()) {
        final AtomicInteger ctr = new AtomicInteger();
        final AtomicReference<Exception> err = new AtomicReference<>();
        int threadCnt = 10;
        for (int i = 0; i < threadCnt; i++) createFile(igfs, new IgfsPath("/file" + i), false);
        multithreaded(new Runnable() {

            @Override
            public void run() {
                int idx = ctr.getAndIncrement();
                IgfsPath path = new IgfsPath("/file" + idx);
                try {
                    byte[][] chunks = new byte[REPEAT_CNT][];
                    for (int i = 0; i < REPEAT_CNT; i++) {
                        chunks[i] = chunk;
                        IgfsOutputStream os = igfs.append(path, false);
                        os.write(chunk);
                        os.close();
                        assert igfs.exists(path);
                    }
                    awaitFileClose(igfs, path);
                    checkFileContent(igfs, path, chunks);
                } catch (IOException | IgniteCheckedException e) {
                    // Log the very first error.
                    err.compareAndSet(null, e);
                }
            }
        }, threadCnt);
        if (err.get() != null)
            throw err.get();
    }
}

Example 93 with IgfsPath

use of org.apache.ignite.igfs.IgfsPath in project ignite by apache.

the class IgfsBlockMessageSystemPoolStarvationSelfTest method testStarvation.

/**
 * Test starvation.
 *
 * @throws Exception If failed.
 */
@SuppressWarnings("unchecked")
public void testStarvation() throws Exception {
    // 1. Create two IGFS file to make all system threads busy.
    CountDownLatch fileWriteLatch = new CountDownLatch(1);
    final IgniteInternalFuture fileFut1 = createFileAsync(new IgfsPath("/file1"), fileWriteLatch);
    final IgniteInternalFuture fileFut2 = createFileAsync(new IgfsPath("/file2"), fileWriteLatch);
    // 2. Start transaction and keep it opened.
    final CountDownLatch txStartLatch = new CountDownLatch(1);
    final CountDownLatch txCommitLatch = new CountDownLatch(1);
    IgniteInternalFuture<Void> txFut = GridTestUtils.runAsync(new Callable<Void>() {

        @Override
        public Void call() throws Exception {
            GridCacheAdapter dataCache = dataCache(attacker);
            try (GridNearTxLocal tx = dataCache.txStartEx(PESSIMISTIC, REPEATABLE_READ)) {
                dataCache.put(DATA_KEY, 0);
                txStartLatch.countDown();
                txCommitLatch.await();
                tx.commit();
            }
            return null;
        }
    });
    txStartLatch.await();
    // 3. Start async operation to drain semaphore permits.
    final IgniteInternalFuture putFut = dataCache(victim).putAsync(DATA_KEY, 1);
    assert !awaitFuture(putFut);
    // 4. Write data to files and ensure we stuck.
    fileWriteLatch.countDown();
    assert !awaitFuture(fileFut1);
    assert !awaitFuture(fileFut2);
    // 5. Finish transaction.
    txCommitLatch.countDown();
    assert awaitFuture(txFut);
    // 6. Async put must succeed.
    assert awaitFuture(putFut);
    // 7. Writes must succeed.
    assert awaitFuture(fileFut1);
    assert awaitFuture(fileFut2);
}

Example 94 with IgfsPath

use of org.apache.ignite.igfs.IgfsPath in project ignite by apache.

the class IgfsDataManagerSelfTest method testDataStoringRemainder.

/**
 * Test file system structure in meta-cache.
 *
 * @throws Exception If failed.
 */
public void testDataStoringRemainder() throws Exception {
    final int blockSize = IGFS_BLOCK_SIZE;
    for (int i = 0; i < 10; i++) {
        IgfsPath path = IgfsPath.ROOT;
        long t = System.currentTimeMillis();
        IgfsEntryInfo info = IgfsUtils.createFile(IgniteUuid.randomUuid(), blockSize, 0L, null, IgfsUtils.DELETE_LOCK_ID, false, null, t, t);
        assertNull(mgr.dataBlock(info, path, 0, null).get());
        byte[] data = new byte[blockSize];
        rnd.nextBytes(data);
        byte[] remainder = new byte[blockSize / 2];
        rnd.nextBytes(remainder);
        info = info.length(info.length() + data.length + remainder.length);
        IgniteInternalFuture<Boolean> fut = mgr.writeStart(info.id());
        IgfsFileAffinityRange range = new IgfsFileAffinityRange();
        byte[] left = mgr.storeDataBlocks(info, info.length(), remainder, remainder.length, ByteBuffer.wrap(data), false, range, null);
        assert left.length == blockSize / 2;
        byte[] remainder2 = new byte[blockSize / 2];
        info = info.length(info.length() + remainder2.length);
        byte[] left2 = mgr.storeDataBlocks(info, info.length(), left, left.length, ByteBuffer.wrap(remainder2), false, range, null);
        assert left2 == null;
        mgr.writeClose(info.id());
        fut.get(3000);
        for (int j = 0; j < NODES_CNT; j++) {
            GridCacheContext<Object, Object> ctx = GridTestUtils.getFieldValue(grid(j).cachex(grid(j).igfsx("igfs").configuration().getDataCacheConfiguration().getName()), "ctx");
            Collection<IgniteInternalTx> txs = ctx.tm().txs();
            assert txs.isEmpty() : "Incomplete transactions: " + txs;
        }
        byte[] concat = U.join(remainder, data, remainder2);
        // Validate data stored in cache.
        for (int pos = 0, block = 0; pos < info.length(); block++) {
            byte[] stored = mgr.dataBlock(info, path, block, null).get();
            assertNotNull("Expects data exist [data.length=" + concat.length + ", block=" + block + ']', stored);
            for (int j = 0; j < stored.length; j++) assertEquals(stored[j], concat[pos + j]);
            pos += stored.length;
        }
        mgr.delete(info);
        long nIters = getTestTimeout() / BUSY_WAIT_SLEEP_INTERVAL;
        assert nIters < Integer.MAX_VALUE;
        boolean rmvBlocks = false;
        // Wait for all blocks to be removed.
        for (int j = 0; j < nIters && !rmvBlocks; j = sleepAndIncrement(BUSY_WAIT_SLEEP_INTERVAL, j)) {
            boolean b = true;
            for (long block = 0; block < info.blocksCount(); block++) b &= mgr.dataBlock(info, path, block, null).get() == null;
            rmvBlocks = b;
        }
        assertTrue("All blocks should be removed from cache.", rmvBlocks);
    }
}

Example 95 with IgfsPath

use of org.apache.ignite.igfs.IgfsPath in project ignite by apache.

the class IgfsDataManagerSelfTest method testDataStoringFlush.

/**
 * @throws Exception If failed.
 */
public void testDataStoringFlush() throws Exception {
    final int blockSize = IGFS_BLOCK_SIZE;
    final int writesCnt = 64;
    for (int i = 0; i < 10; i++) {
        IgfsPath path = IgfsPath.ROOT;
        long t = System.currentTimeMillis();
        IgfsEntryInfo info = IgfsUtils.createFile(IgniteUuid.randomUuid(), blockSize, 0L, null, IgfsUtils.DELETE_LOCK_ID, false, null, t, t);
        IgfsFileAffinityRange range = new IgfsFileAffinityRange();
        assertNull(mgr.dataBlock(info, path, 0, null).get());
        int chunkSize = blockSize / 4;
        byte[] data = new byte[chunkSize];
        info = info.length(info.length() + data.length * writesCnt);
        IgniteInternalFuture<Boolean> fut = mgr.writeStart(info.id());
        for (int j = 0; j < 64; j++) {
            Arrays.fill(data, (byte) (j / 4));
            byte[] left = mgr.storeDataBlocks(info, (j + 1) * chunkSize, null, 0, ByteBuffer.wrap(data), true, range, null);
            assert left == null : "No remainder should be returned if flush is true: " + Arrays.toString(left);
        }
        mgr.writeClose(info.id());
        assertTrue(range.regionEqual(new IgfsFileAffinityRange(0, writesCnt * chunkSize - 1, null)));
        fut.get(3000);
        for (int j = 0; j < NODES_CNT; j++) {
            GridCacheContext<Object, Object> ctx = GridTestUtils.getFieldValue(grid(j).cachex(grid(j).igfsx("igfs").configuration().getDataCacheConfiguration().getName()), "ctx");
            Collection<IgniteInternalTx> txs = ctx.tm().txs();
            assert txs.isEmpty() : "Incomplete transactions: " + txs;
        }
        // Validate data stored in cache.
        for (int pos = 0, block = 0; pos < info.length(); block++) {
            byte[] stored = mgr.dataBlock(info, path, block, null).get();
            assertNotNull("Expects data exist [block=" + block + ']', stored);
            for (byte b : stored) assertEquals(b, (byte) block);
            pos += stored.length;
        }
        IgniteInternalFuture<Object> delFut = mgr.delete(info);
        delFut.get();
        for (long block = 0; block < info.blocksCount(); block++) assertNull(mgr.dataBlock(info, path, block, null).get());
    }
}