Examples with IgniteFileSystem org.apache.ignite.IgniteFileSystem used on opensource projects

Example 1 with IgniteFileSystem

use of org.apache.ignite.IgniteFileSystem in project ignite by apache.

the class IgfsExample method main.

/**
 * Executes example.
 *
 * @param args Command line arguments, none required.
 * @throws Exception If example execution failed.
 */
public static void main(String[] args) throws Exception {
    Ignite ignite = Ignition.start("examples/config/filesystem/example-igfs.xml");
    System.out.println();
    System.out.println(">>> IGFS example started.");
    try {
        // Get an instance of Ignite File System.
        IgniteFileSystem fs = ignite.fileSystem("igfs");
        // Working directory path.
        IgfsPath workDir = new IgfsPath("/examples/fs");
        // Cleanup working directory.
        delete(fs, workDir);
        // Create empty working directory.
        mkdirs(fs, workDir);
        // Print information for working directory.
        printInfo(fs, workDir);
        // File path.
        IgfsPath filePath = new IgfsPath(workDir, "file.txt");
        // Create file.
        create(fs, filePath, new byte[] { 1, 2, 3 });
        // Print information for file.
        printInfo(fs, filePath);
        // Append more data to previously created file.
        append(fs, filePath, new byte[] { 4, 5 });
        // Print information for file.
        printInfo(fs, filePath);
        // Read data from file.
        read(fs, filePath);
        // Delete file.
        delete(fs, filePath);
        // Print information for file.
        printInfo(fs, filePath);
        // Create several files.
        for (int i = 0; i < 5; i++) create(fs, new IgfsPath(workDir, "file-" + i + ".txt"), null);
        list(fs, workDir);
    } finally {
        Ignition.stop(false);
    }
}

Example 2 with IgniteFileSystem

use of org.apache.ignite.IgniteFileSystem in project ignite by apache.

the class IgfsTask method map.

/**
 * {@inheritDoc}
 */
@Nullable
@Override
public final Map<? extends ComputeJob, ClusterNode> map(List<ClusterNode> subgrid, @Nullable IgfsTaskArgs<T> args) {
    assert ignite != null;
    assert args != null;
    IgniteFileSystem fs = ignite.fileSystem(args.igfsName());
    IgfsProcessorAdapter igfsProc = ((IgniteKernal) ignite).context().igfs();
    Map<ComputeJob, ClusterNode> splitMap = new HashMap<>();
    Map<UUID, ClusterNode> nodes = mapSubgrid(subgrid);
    for (IgfsPath path : args.paths()) {
        IgfsFile file = fs.info(path);
        if (file == null) {
            if (args.skipNonExistentFiles())
                continue;
            else
                throw new IgniteException("Failed to process IGFS file because it doesn't exist: " + path);
        }
        Collection<IgfsBlockLocation> aff = fs.affinity(path, 0, file.length(), args.maxRangeLength());
        long totalLen = 0;
        for (IgfsBlockLocation loc : aff) {
            ClusterNode node = null;
            for (UUID nodeId : loc.nodeIds()) {
                node = nodes.get(nodeId);
                if (node != null)
                    break;
            }
            if (node == null)
                throw new IgniteException("Failed to find any of block affinity nodes in subgrid [loc=" + loc + ", subgrid=" + subgrid + ']');
            IgfsJob job = createJob(path, new IgfsFileRange(file.path(), loc.start(), loc.length()), args);
            if (job != null) {
                ComputeJob jobImpl = igfsProc.createJob(job, fs.name(), file.path(), loc.start(), loc.length(), args.recordResolver());
                splitMap.put(jobImpl, node);
            }
            totalLen += loc.length();
        }
        assert totalLen == file.length();
    }
    return splitMap;
}

Example 3 with IgniteFileSystem

use of org.apache.ignite.IgniteFileSystem in project ignite by apache.

the class IgfsSecondaryFileSystemInjectionSelfTest method testInjectPrimaryByField.

/**
 * @throws Exception If failed.
 */
@SuppressWarnings({ "UnusedDeclaration" })
public void testInjectPrimaryByField() throws Exception {
    secondary = new TestBaseSecondaryFsMock() {

        @FileSystemResource
        private IgfsImpl igfs;

        @LoggerResource
        private IgniteLogger log;

        @IgniteInstanceResource
        private Ignite ig;

        @Override
        boolean checkInjection(Ignite ignite, IgniteFileSystem primary) {
            return igfs == primary && log instanceof IgniteLogger && ig == ignite;
        }
    };
    Ignite ig = startGrid(0);
    IgniteFileSystem igfs = ig.fileSystem(IGFS_NAME);
    assert secondary.checkInjection(ig, igfs);
}

Example 4 with IgniteFileSystem

use of org.apache.ignite.IgniteFileSystem in project ignite by apache.

the class IgfsSecondaryFileSystemInjectionSelfTest method testInjectPrimaryByMethods.

/**
 * @throws Exception If failed.
 */
@SuppressWarnings({ "UnusedDeclaration" })
public void testInjectPrimaryByMethods() throws Exception {
    secondary = new TestBaseSecondaryFsMock() {

        /**
         * Ignite instance.
         */
        private Ignite ig;

        /**
         * IGFS instance.
         */
        private IgniteFileSystem igfs;

        /**
         * Logger injected flag
         */
        private boolean logSet;

        /**
         * @param igfs Primary IGFS.
         */
        @FileSystemResource
        void setPrimaryIgfs(IgfsImpl igfs) {
            this.igfs = igfs;
        }

        /**
         * @param log Ignite logger.
         */
        @LoggerResource
        void setIgLogger(IgniteLogger log) {
            logSet = log instanceof IgniteLogger;
        }

        /**
         * @param ig Ignite instance.
         */
        @IgniteInstanceResource
        void setIgniteInst(Ignite ig) {
            this.ig = ig;
        }

        @Override
        boolean checkInjection(Ignite ignite, IgniteFileSystem primary) {
            return ignite == ig && primary == igfs && logSet;
        }
    };
    Ignite ig = startGrid(0);
    IgniteFileSystem igfs = ig.fileSystem(IGFS_NAME);
    assert secondary.checkInjection(ig, igfs);
}

Example 5 with IgniteFileSystem

use of org.apache.ignite.IgniteFileSystem in project ignite by apache.

the class IgfsStreamsSelfTest method testCreateFileFragmented.

/**
 * @throws Exception If failed.
 */
public void testCreateFileFragmented() throws Exception {
    IgfsEx impl = (IgfsEx) grid(0).fileSystem("igfs");
    String metaCacheName = grid(0).igfsx("igfs").configuration().getMetaCacheConfiguration().getName();
    final String dataCacheName = grid(0).igfsx("igfs").configuration().getDataCacheConfiguration().getName();
    IgfsFragmentizerManager fragmentizer = impl.context().fragmentizer();
    GridTestUtils.setFieldValue(fragmentizer, "fragmentizerEnabled", false);
    IgfsPath path = new IgfsPath("/file");
    try {
        IgniteFileSystem fs0 = grid(0).fileSystem("igfs");
        IgniteFileSystem fs1 = grid(1).fileSystem("igfs");
        IgniteFileSystem fs2 = grid(2).fileSystem("igfs");
        try (IgfsOutputStream out = fs0.create(path, 128, false, 1, CFG_GRP_SIZE, F.asMap(IgfsUtils.PROP_PREFER_LOCAL_WRITES, "true"))) {
            // 1.5 blocks
            byte[] data = new byte[CFG_BLOCK_SIZE * 3 / 2];
            Arrays.fill(data, (byte) 1);
            out.write(data);
        }
        try (IgfsOutputStream out = fs1.append(path, false)) {
            // 1.5 blocks.
            byte[] data = new byte[CFG_BLOCK_SIZE * 3 / 2];
            Arrays.fill(data, (byte) 2);
            out.write(data);
        }
        // After this we should have first two block colocated with grid 0 and last block colocated with grid 1.
        IgfsFileImpl fileImpl = (IgfsFileImpl) fs.info(path);
        GridCacheAdapter<Object, Object> metaCache = ((IgniteKernal) grid(0)).internalCache(metaCacheName);
        IgfsEntryInfo fileInfo = (IgfsEntryInfo) metaCache.get(fileImpl.fileId());
        IgfsFileMap map = fileInfo.fileMap();
        List<IgfsFileAffinityRange> ranges = map.ranges();
        assertEquals(2, ranges.size());
        assertTrue(ranges.get(0).startOffset() == 0);
        assertTrue(ranges.get(0).endOffset() == 2 * CFG_BLOCK_SIZE - 1);
        assertTrue(ranges.get(1).startOffset() == 2 * CFG_BLOCK_SIZE);
        assertTrue(ranges.get(1).endOffset() == 3 * CFG_BLOCK_SIZE - 1);
        // Validate data read after colocated writes.
        try (IgfsInputStream in = fs2.open(path)) {
            // Validate first part of file.
            for (int i = 0; i < CFG_BLOCK_SIZE * 3 / 2; i++) assertEquals((byte) 1, in.read());
            // Validate second part of file.
            for (int i = 0; i < CFG_BLOCK_SIZE * 3 / 2; i++) assertEquals((byte) 2, in.read());
            assertEquals(-1, in.read());
        }
    } finally {
        GridTestUtils.setFieldValue(fragmentizer, "fragmentizerEnabled", true);
        boolean hasData = false;
        for (int i = 0; i < NODES_CNT; i++) hasData |= !grid(i).cachex(dataCacheName).isEmpty();
        assertTrue(hasData);
        fs.delete(path, true);
    }
    GridTestUtils.retryAssert(log, ASSERT_RETRIES, ASSERT_RETRY_INTERVAL, new CAX() {

        @Override
        public void applyx() {
            for (int i = 0; i < NODES_CNT; i++) assertTrue(grid(i).cachex(dataCacheName).isEmpty());
        }
    });
}