Examples with FileIOFactory org.apache.ignite.internal.processors.cache.persistence.file.FileIOFactory used on opensource projects

Example 1 with FileIOFactory

use of org.apache.ignite.internal.processors.cache.persistence.file.FileIOFactory in project ignite by apache.

the class GridIoManagerFileTransmissionSelfTest method testFileHandlerReconnectOnReadFail.

/**
 * @throws Exception If fails.
 */
@Test(expected = IgniteCheckedException.class)
public void testFileHandlerReconnectOnReadFail() throws Exception {
    final String chunkDownloadExMsg = "Test exception. Chunk processing error.";
    snd = startGrid(0);
    rcv = startGrid(1);
    File fileToSend = createFileRandomData("testFile", 5 * 1024 * 1024);
    final AtomicInteger readChunks = new AtomicInteger();
    transmissionFileIoFactory(rcv, new FileIOFactory() {

        @Override
        public FileIO create(File file, OpenOption... modes) throws IOException {
            fileIo[0] = IO_FACTORY.create(file, modes);
            // Blocking writer and stopping node FileIo.
            return new FileIODecorator(fileIo[0]) {

                @Override
                public long transferFrom(ReadableByteChannel src, long position, long count) throws IOException {
                    // Read 4 chunks than throw an exception to emulate error processing.
                    if (readChunks.incrementAndGet() == 4)
                        throw new IgniteException(chunkDownloadExMsg);
                    return super.transferFrom(src, position, count);
                }
            };
        }
    });
    rcv.context().io().addTransmissionHandler(topic, new DefaultTransmissionHandler(rcv, fileToSend, tempStore) {

        @Override
        public void onException(UUID nodeId, Throwable err) {
            assertEquals(chunkDownloadExMsg, err.getMessage());
        }
    });
    try (GridIoManager.TransmissionSender sender = snd.context().io().openTransmissionSender(rcv.localNode().id(), topic)) {
        sender.send(fileToSend, TransmissionPolicy.FILE);
    }
}

Example 2 with FileIOFactory

use of org.apache.ignite.internal.processors.cache.persistence.file.FileIOFactory in project ignite by apache.

the class GridCommandHandlerDefragmentationTest method testDefragmentationCancelInProgress.

/**
 * @throws Exception If failed.
 */
@Test
public void testDefragmentationCancelInProgress() throws Exception {
    IgniteEx ig = startGrid(0);
    ig.cluster().state(ClusterState.ACTIVE);
    IgniteCache<Object, Object> cache = ig.getOrCreateCache(DEFAULT_CACHE_NAME);
    for (int i = 0; i < 1024; i++) cache.put(i, i);
    forceCheckpoint(ig);
    String grid0ConsId = ig.configuration().getConsistentId().toString();
    ListeningTestLogger testLog = new ListeningTestLogger();
    CommandHandler cmd = createCommandHandler(testLog);
    assertEquals(EXIT_CODE_OK, execute(cmd, "--defragmentation", "schedule", "--nodes", grid0ConsId));
    String port = grid(0).localNode().attribute(IgniteNodeAttributes.ATTR_REST_TCP_PORT).toString();
    stopGrid(0);
    blockCdl = new CountDownLatch(128);
    UnaryOperator<IgniteConfiguration> cfgOp = cfg -> {
        DataStorageConfiguration dsCfg = cfg.getDataStorageConfiguration();
        FileIOFactory delegate = dsCfg.getFileIOFactory();
        dsCfg.setFileIOFactory((file, modes) -> {
            if (file.getName().contains("dfrg")) {
                if (blockCdl.getCount() == 0) {
                    try {
                        // Slow down defragmentation process.
                        // This'll be enough for the test since we have, like, 900 partitions left.
                        Thread.sleep(100);
                    } catch (InterruptedException ignore) {
                    // No-op.
                    }
                } else
                    blockCdl.countDown();
            }
            return delegate.create(file, modes);
        });
        return cfg;
    };
    IgniteInternalFuture<?> fut = GridTestUtils.runAsync(() -> {
        try {
            startGrid(0, cfgOp);
        } catch (Exception e) {
            // No-op.
            throw new RuntimeException(e);
        }
    });
    blockCdl.await();
    LogListener logLsnr = LogListener.matches("Defragmentation cancelled successfully.").build();
    testLog.registerListener(logLsnr);
    assertEquals(EXIT_CODE_OK, execute(cmd, "--port", port, "--defragmentation", "cancel"));
    assertTrue(logLsnr.check());
    fut.get();
    testLog.clearListeners();
    logLsnr = LogListener.matches("Defragmentation is already completed or has been cancelled previously.").build();
    testLog.registerListener(logLsnr);
    assertEquals(EXIT_CODE_OK, execute(cmd, "--port", port, "--defragmentation", "cancel"));
    assertTrue(logLsnr.check());
}

Example 3 with FileIOFactory

use of org.apache.ignite.internal.processors.cache.persistence.file.FileIOFactory in project ignite by apache.

the class IgniteWalIteratorFactory method resolveWalFiles.

/**
 * This methods checks all provided files to be correct WAL segment.
 * Header record and its position is checked. WAL position is used to determine real index.
 * File index from file name is ignored.
 *
 * @param iteratorParametersBuilder IteratorParametersBuilder.
 * @return list of file descriptors with checked header records, having correct file index is set
 */
public List<FileDescriptor> resolveWalFiles(IteratorParametersBuilder iteratorParametersBuilder) {
    File[] filesOrDirs = iteratorParametersBuilder.filesOrDirs;
    if (filesOrDirs == null || filesOrDirs.length == 0)
        return Collections.emptyList();
    final FileIOFactory ioFactory = iteratorParametersBuilder.ioFactory;
    final TreeSet<FileDescriptor> descriptors = new TreeSet<>();
    for (File file : filesOrDirs) {
        if (file.isDirectory()) {
            try {
                walkFileTree(file.toPath(), new SimpleFileVisitor<Path>() {

                    @Override
                    public FileVisitResult visitFile(Path path, BasicFileAttributes attrs) {
                        addFileDescriptor(path.toFile(), ioFactory, descriptors);
                        return FileVisitResult.CONTINUE;
                    }
                });
            } catch (IOException e) {
                U.error(log, "Failed to walk directories from root [" + file + "]. Skipping this directory.", e);
            }
            continue;
        }
        addFileDescriptor(file, ioFactory, descriptors);
    }
    return new ArrayList<>(descriptors);
}

Example 4 with FileIOFactory

use of org.apache.ignite.internal.processors.cache.persistence.file.FileIOFactory in project ignite by apache.

the class IgnitePdsDefragmentationTest method startAndAwaitNodeFail.

/**
 * @throws IgniteInterruptedCheckedException If fail.
 */
private void startAndAwaitNodeFail(File workDir) throws IgniteInterruptedCheckedException {
    String errMsg = "Failed to create defragmentation completion marker.";
    AtomicBoolean errOccurred = new AtomicBoolean();
    UnaryOperator<IgniteConfiguration> cfgOp = cfg -> {
        DataStorageConfiguration dsCfg = cfg.getDataStorageConfiguration();
        FileIOFactory delegate = dsCfg.getFileIOFactory();
        dsCfg.setFileIOFactory((file, modes) -> {
            if (file.equals(defragmentationCompletionMarkerFile(workDir))) {
                errOccurred.set(true);
                throw new IOException(errMsg);
            }
            return delegate.create(file, modes);
        });
        return cfg;
    };
    AtomicBoolean nodeStopped = new AtomicBoolean();
    IgnitionListener nodeStopListener = (name, state) -> {
        if (name.equals(getTestIgniteInstanceName(0)) && state == IgniteState.STOPPED_ON_FAILURE)
            nodeStopped.set(true);
    };
    Ignition.addListener(nodeStopListener);
    try {
        try {
            startGrid(0, cfgOp);
        } catch (Exception ignore) {
        // No-op.
        }
        // Failed node can leave interrupted status of the thread that needs to be cleared,
        // otherwise following "wait" wouldn't work.
        // This call can't be moved inside of "catch" block because interruption can actually be silent.
        Thread.interrupted();
        assertTrue(GridTestUtils.waitForCondition(errOccurred::get, 3_000L));
        assertTrue(GridTestUtils.waitForCondition(nodeStopped::get, 3_000L));
    } finally {
        Ignition.removeListener(nodeStopListener);
    }
}

Example 5 with FileIOFactory

use of org.apache.ignite.internal.processors.cache.persistence.file.FileIOFactory in project ignite by apache.

the class IgniteSequentialNodeCrashRecoveryTest method testCrashOnCheckpointAfterLogicalRecovery.

/**
 * @throws Exception if failed.
 */
@Test
public void testCrashOnCheckpointAfterLogicalRecovery() throws Exception {
    IgniteEx g = startGrid(0);
    g.cluster().active(true);
    g.getOrCreateCache(new CacheConfiguration<>("cache").setAtomicityMode(CacheAtomicityMode.TRANSACTIONAL).setAffinity(new RendezvousAffinityFunction(false, 8)));
    disableCheckpoints(g);
    {
        IgniteCache<Object, Object> cache = g.cache("cache");
        // Now that checkpoints are disabled, put some data to the cache.
        GridTestUtils.runMultiThreaded(() -> {
            for (int i = 0; i < 400; i++) cache.put(i % 100, Thread.currentThread().getName());
        }, 64, "update-thread");
    }
    Collection<FullPageId> dirtyAfterLoad = captureDirtyPages(g);
    stopGrid(0);
    CheckpointFailingIoFactory f = (CheckpointFailingIoFactory) (fileIoFactory = new CheckpointFailingIoFactory(false));
    StopLatchFailureHandler fh = (StopLatchFailureHandler) (failureHnd = new StopLatchFailureHandler());
    // Blocking first exchange to prevent checkpoint on node start(reason = 'node started').
    BlockingDiscoverySpi ds = (BlockingDiscoverySpi) (discoverySpi = new BlockingDiscoverySpi((m) -> m instanceof ChangeGlobalStateMessage));
    // Now start the node. Since the checkpoint was disabled, logical recovery will be performed.
    g = startGrid(0);
    fileIoFactory = null;
    failureHnd = null;
    // Capture dirty pages after logical recovery & updates.
    Collection<FullPageId> dirtyAfterRecoveryAndUpdates = captureDirtyPages(g);
    f.startFailing();
    ds.clearBlock();
    triggerCheckpoint(g);
    assertTrue("Failed to wait for checkpoint failure", fh.waitFailed());
    // Capture pages we marked on first run and did not mark on second run.
    dirtyAfterLoad.removeAll(dirtyAfterRecoveryAndUpdates);
    assertFalse(dirtyAfterLoad.isEmpty());
    fileIoFactory = new CheckingIoFactory(dirtyAfterLoad);
    discoverySpi = null;
    g = startGrid(0);
    {
        IgniteCache<Object, Object> cache = g.cache("cache");
        for (int i = 0; i < 400; i++) cache.put(100 + (i % 100), Thread.currentThread().getName());
        for (int i = 0; i < 200; i++) assertTrue("i=" + i, cache.containsKey(i));
    }
}