Examples with FileIOFactory org.apache.ignite.internal.processors.cache.persistence.file.FileIOFactory used on opensource projects

Example 11 with FileIOFactory

use of org.apache.ignite.internal.processors.cache.persistence.file.FileIOFactory in project ignite by apache.

the class GridCacheDatabaseSharedManager method resolvePageSizeFromPartitionFile.

/**
 * @param partFile Partition file.
 */
private int resolvePageSizeFromPartitionFile(Path partFile) throws IOException, IgniteCheckedException {
    FileIOFactory ioFactory = persistenceCfg.getFileIOFactory();
    try (FileIO fileIO = ioFactory.create(partFile.toFile())) {
        int minimalHdr = FilePageStore.HEADER_SIZE;
        if (fileIO.size() < minimalHdr)
            throw new IgniteCheckedException("Partition file is too small: " + partFile);
        ByteBuffer hdr = ByteBuffer.allocate(minimalHdr).order(ByteOrder.nativeOrder());
        fileIO.readFully(hdr);
        hdr.rewind();
        // Read signature.
        hdr.getLong();
        // Read version.
        hdr.getInt();
        // Read type.
        hdr.get();
        int pageSize = hdr.getInt();
        if (pageSize == 2048) {
            U.quietAndWarn(log, "You are currently using persistent store with 2K pages (DataStorageConfiguration#" + "pageSize). If you use SSD disk, consider migrating to 4K pages for better IO performance.");
        }
        return pageSize;
    }
}

Example 12 with FileIOFactory

use of org.apache.ignite.internal.processors.cache.persistence.file.FileIOFactory in project ignite by apache.

the class GridCommandHandlerDefragmentationTest method testDefragmentationStatus.

/**
 * @throws Exception If failed.
 */
@Test
public void testDefragmentationStatus() throws Exception {
    IgniteEx ig = startGrid(0);
    ig.cluster().state(ClusterState.ACTIVE);
    ig.getOrCreateCache(DEFAULT_CACHE_NAME + "1");
    IgniteCache<Object, Object> cache = ig.getOrCreateCache(DEFAULT_CACHE_NAME + "2");
    ig.getOrCreateCache(DEFAULT_CACHE_NAME + "3");
    for (int i = 0; i < 1024; i++) cache.put(i, i);
    forceCheckpoint(ig);
    String grid0ConsId = ig.configuration().getConsistentId().toString();
    ListeningTestLogger testLog = new ListeningTestLogger();
    CommandHandler cmd = createCommandHandler(testLog);
    assertEquals(EXIT_CODE_OK, execute(cmd, "--defragmentation", "schedule", "--nodes", grid0ConsId));
    String port = grid(0).localNode().attribute(IgniteNodeAttributes.ATTR_REST_TCP_PORT).toString();
    stopGrid(0);
    blockCdl = new CountDownLatch(128);
    waitCdl = new CountDownLatch(1);
    UnaryOperator<IgniteConfiguration> cfgOp = cfg -> {
        DataStorageConfiguration dsCfg = cfg.getDataStorageConfiguration();
        FileIOFactory delegate = dsCfg.getFileIOFactory();
        dsCfg.setFileIOFactory((file, modes) -> {
            if (file.getName().contains("dfrg")) {
                if (blockCdl.getCount() == 0) {
                    try {
                        waitCdl.await();
                    } catch (InterruptedException ignore) {
                    // No-op.
                    }
                } else
                    blockCdl.countDown();
            }
            return delegate.create(file, modes);
        });
        return cfg;
    };
    IgniteInternalFuture<?> fut = GridTestUtils.runAsync(() -> {
        try {
            startGrid(0, cfgOp);
        } catch (Exception e) {
            // No-op.
            throw new RuntimeException(e);
        }
    });
    blockCdl.await();
    List<LogListener> logLsnrs = Arrays.asList(LogListener.matches("default1 - size before/after: 0MB/0MB").build(), LogListener.matches("default2 - partitions processed/all:").build(), LogListener.matches("Awaiting defragmentation: default3").build());
    for (LogListener logLsnr : logLsnrs) testLog.registerListener(logLsnr);
    assertEquals(EXIT_CODE_OK, execute(cmd, "--port", port, "--defragmentation", "status"));
    waitCdl.countDown();
    for (LogListener logLsnr : logLsnrs) assertTrue(logLsnr.check());
    fut.get();
    ((GridCacheDatabaseSharedManager) grid(0).context().cache().context().database()).defragmentationManager().completionFuture().get();
    testLog.clearListeners();
    logLsnrs = Arrays.asList(LogListener.matches("default1 - size before/after: 0MB/0MB").build(), LogListener.matches(Pattern.compile("default2 - size before/after: (\\S+)/\\1")).build(), LogListener.matches("default3 - size before/after: 0MB/0MB").build());
    for (LogListener logLsnr : logLsnrs) testLog.registerListener(logLsnr);
    assertEquals(EXIT_CODE_OK, execute(cmd, "--port", port, "--defragmentation", "status"));
    for (LogListener logLsnr : logLsnrs) assertTrue(logLsnr.check());
}

Example 13 with FileIOFactory

use of org.apache.ignite.internal.processors.cache.persistence.file.FileIOFactory in project ignite by apache.

the class GridIoManagerFileTransmissionSelfTest method tesFileHandlerReconnectTimeouted.

/**
 * @throws Exception If fails.
 */
@Test
public void tesFileHandlerReconnectTimeouted() throws Exception {
    rcv = startGrid(1);
    snd = startGrid(0);
    final AtomicInteger chunksCnt = new AtomicInteger();
    final CountDownLatch sndLatch = ((BlockingOpenChannelCommunicationSpi) snd.context().config().getCommunicationSpi()).latch;
    final AtomicReference<Throwable> refErr = new AtomicReference<>();
    File fileToSend = createFileRandomData("testFile", 5 * 1024 * 1024);
    transmissionFileIoFactory(snd, new FileIOFactory() {

        @Override
        public FileIO create(File file, OpenOption... modes) throws IOException {
            FileIO fileIo = IO_FACTORY.create(file, modes);
            return new FileIODecorator(fileIo) {

                /**
                 * {@inheritDoc}
                 */
                @Override
                public long transferTo(long position, long count, WritableByteChannel target) throws IOException {
                    if (chunksCnt.incrementAndGet() == 10) {
                        target.close();
                        ((BlockingOpenChannelCommunicationSpi) snd.context().config().getCommunicationSpi()).block = true;
                    }
                    return super.transferTo(position, count, target);
                }
            };
        }
    });
    rcv.context().io().addTransmissionHandler(topic, new DefaultTransmissionHandler(rcv, fileToSend, tempStore) {

        @Override
        public void onException(UUID nodeId, Throwable err) {
            refErr.compareAndSet(null, err);
            sndLatch.countDown();
        }
    });
    try (GridIoManager.TransmissionSender sender = snd.context().io().openTransmissionSender(rcv.localNode().id(), topic)) {
        sender.send(fileToSend, TransmissionPolicy.FILE);
    } catch (IgniteCheckedException | IOException | InterruptedException e) {
        // Ignore err.
        U.warn(log, e);
    }
    assertNotNull("Timeout exception not occurred", refErr.get());
    assertEquals("Type of timeout exception incorrect: " + refErr.get(), IgniteCheckedException.class, refErr.get().getClass());
}

Example 14 with FileIOFactory

use of org.apache.ignite.internal.processors.cache.persistence.file.FileIOFactory in project ignite by apache.

the class GridIoManagerFileTransmissionSelfTest method testChunkHandlerWithReconnect.

/**
 * @throws Exception If fails.
 */
@Test
public void testChunkHandlerWithReconnect() throws Exception {
    snd = startGrid(0);
    rcv = startGrid(1);
    final String filePrefix = "testFile";
    final AtomicInteger cnt = new AtomicInteger();
    final AtomicInteger acceptedChunks = new AtomicInteger();
    final File file = new File(tempStore, filePrefix + "_" + rcv.localNode().id());
    File fileToSend = createFileRandomData(filePrefix, 10 * 1024 * 1024);
    transmissionFileIoFactory(snd, new FileIOFactory() {

        @Override
        public FileIO create(File file, OpenOption... modes) throws IOException {
            FileIO fileIo = IO_FACTORY.create(file, modes);
            return new FileIODecorator(fileIo) {

                /**
                 * {@inheritDoc}
                 */
                @Override
                public long transferTo(long position, long count, WritableByteChannel target) throws IOException {
                    // Send 5 chunks and close the channel.
                    if (cnt.incrementAndGet() == 10)
                        target.close();
                    return super.transferTo(position, count, target);
                }
            };
        }
    });
    rcv.context().io().addTransmissionHandler(topic, new TransmissionHandlerAdapter() {

        /**
         * {@inheritDoc}
         */
        @Override
        public void onEnd(UUID rmtNodeId) {
            U.closeQuiet(fileIo[0]);
            fileIo[0] = null;
        }

        /**
         * {@inheritDoc}
         */
        @Override
        public void onException(UUID nodeId, Throwable err) {
            U.closeQuiet(fileIo[0]);
            fileIo[0] = null;
        }

        /**
         * {@inheritDoc}
         */
        @Override
        public Consumer<ByteBuffer> chunkHandler(UUID nodeId, TransmissionMeta initMeta) {
            if (fileIo[0] == null) {
                try {
                    fileIo[0] = IO_FACTORY.create(file);
                    fileIo[0].position(initMeta.offset());
                } catch (IOException e) {
                    throw new IgniteException(e);
                }
            }
            return new Consumer<ByteBuffer>() {

                @Override
                public void accept(ByteBuffer buff) {
                    try {
                        assertTrue(buff.order() == ByteOrder.nativeOrder());
                        assertEquals(0, buff.position());
                        assertEquals(buff.limit(), buff.capacity());
                        fileIo[0].writeFully(buff);
                        acceptedChunks.getAndIncrement();
                    } catch (Throwable e) {
                        throw new IgniteException(e);
                    }
                }
            };
        }
    });
    try (GridIoManager.TransmissionSender sender = snd.context().io().openTransmissionSender(rcv.localNode().id(), topic)) {
        sender.send(fileToSend, TransmissionPolicy.CHUNK);
    }
    assertEquals("Remote node must accept all chunks", fileToSend.length() / rcv.configuration().getDataStorageConfiguration().getPageSize(), acceptedChunks.get());
    assertEquals("Received file and sent files have not the same lengtgh", fileToSend.length(), file.length());
    assertCrcEquals(fileToSend, file);
    assertNull(fileIo[0]);
}

Example 15 with FileIOFactory

use of org.apache.ignite.internal.processors.cache.persistence.file.FileIOFactory in project ignite by apache.

the class GridIoManagerFileTransmissionSelfTest method testFileHandlerCleanedUpIfSenderLeft.

/**
 * @throws Exception If fails.
 */
@Test
public void testFileHandlerCleanedUpIfSenderLeft() throws Exception {
    snd = startGrid(0);
    rcv = startGrid(1);
    File fileToSend = createFileRandomData("tempFile15Mb", 15 * 1024 * 1024);
    File downloadTo = U.resolveWorkDirectory(tempStore.getAbsolutePath(), "download", true);
    transmissionFileIoFactory(snd, new FileIOFactory() {

        @Override
        public FileIO create(File file, OpenOption... modes) throws IOException {
            FileIO fileIo = IO_FACTORY.create(file, modes);
            return new FileIODecorator(fileIo) {

                /**
                 * {@inheritDoc}
                 */
                @Override
                public long transferTo(long position, long count, WritableByteChannel target) throws IOException {
                    long transferred = super.transferTo(position, count, target);
                    stopGrid(snd.name(), true);
                    return transferred;
                }
            };
        }
    });
    rcv.context().io().addTransmissionHandler(topic, new DefaultTransmissionHandler(rcv, fileToSend, tempStore) {

        /**
         * {@inheritDoc}
         */
        @Override
        public String filePath(UUID nodeId, TransmissionMeta fileMeta) {
            return new File(downloadTo, fileMeta.name()).getAbsolutePath();
        }
    });
    Exception err = null;
    try (GridIoManager.TransmissionSender sender = snd.context().io().openTransmissionSender(rcv.localNode().id(), topic)) {
        sender.send(fileToSend, TransmissionPolicy.FILE);
    } catch (Exception e) {
        // Ignore node stopping exception.
        err = e;
    }
    assertEquals(NodeStoppingException.class, err.getClass());
    assertEquals("Incomplete resources must be cleaned up on sender left", 0, fileCount(downloadTo.toPath()));
}