Examples with FileIOFactory org.apache.ignite.internal.processors.cache.persistence.file.FileIOFactory used on opensource projects

Example 16 with FileIOFactory

use of org.apache.ignite.internal.processors.cache.persistence.file.FileIOFactory in project ignite by apache.

the class GridIoManagerFileTransmissionSelfTest method testFileHandlerOnReceiverLeft.

/**
 * @throws Exception If fails.
 */
@Test(expected = IgniteCheckedException.class)
public void testFileHandlerOnReceiverLeft() throws Exception {
    final int fileSizeBytes = 5 * 1024 * 1024;
    final AtomicInteger chunksCnt = new AtomicInteger();
    snd = startGrid(0);
    rcv = startGrid(1);
    File fileToSend = createFileRandomData("testFile", fileSizeBytes);
    transmissionFileIoFactory(snd, new FileIOFactory() {

        @Override
        public FileIO create(File file, OpenOption... modes) throws IOException {
            FileIO fileIo = IO_FACTORY.create(file, modes);
            // Blocking writer and stopping node FileIo.
            return new FileIODecorator(fileIo) {

                /**
                 * {@inheritDoc}
                 */
                @Override
                public long transferTo(long position, long count, WritableByteChannel target) throws IOException {
                    // Send 5 chunks than stop the rcv.
                    if (chunksCnt.incrementAndGet() == 5)
                        stopGrid(rcv.name(), true);
                    return super.transferTo(position, count, target);
                }
            };
        }
    });
    rcv.context().io().addTransmissionHandler(topic, new DefaultTransmissionHandler(rcv, fileToSend, tempStore));
    try (GridIoManager.TransmissionSender sender = snd.context().io().openTransmissionSender(rcv.localNode().id(), topic)) {
        sender.send(fileToSend, TransmissionPolicy.FILE);
    }
}

Example 17 with FileIOFactory

use of org.apache.ignite.internal.processors.cache.persistence.file.FileIOFactory in project ignite by apache.

the class IgniteWalIteratorSwitchSegmentTest method checkInvariantSwitchSegment.

/**
 * @param serVer WAL serializer version.
 * @throws Exception If some thing failed.
 */
private void checkInvariantSwitchSegment(int serVer) throws Exception {
    String workDir = U.defaultWorkDirectory();
    T2<IgniteWriteAheadLogManager, RecordSerializer> initTup = initiate(serVer, workDir);
    IgniteWriteAheadLogManager walMgr = initTup.get1();
    RecordSerializer recordSerializer = initTup.get2();
    int switchSegmentRecordSize = recordSerializer.size(new SwitchSegmentRecord());
    log.info("switchSegmentRecordSize:" + switchSegmentRecordSize);
    int tailSize = 0;
    /* Initial record payload size. */
    int payloadSize = 1024;
    int recSize = 0;
    MetastoreDataRecord rec = null;
    /* Record size. */
    int recordTypeSize = 1;
    /* Record pointer. */
    int recordPointerSize = 8 + 4 + 4;
    int lowBound = recordTypeSize + recordPointerSize;
    int highBound = lowBound + /*CRC*/
    4;
    int attempt = 1000;
    // Try find how many record need for specific tail size.
    while (true) {
        if (attempt < 0)
            throw new IgniteCheckedException("Can not find any payload size for test, " + "lowBound=" + lowBound + ", highBound=" + highBound);
        if (tailSize >= lowBound && tailSize < highBound)
            break;
        payloadSize++;
        byte[] payload = new byte[payloadSize];
        // Fake record for payload.
        rec = new MetastoreDataRecord("0", payload);
        recSize = recordSerializer.size(rec);
        tailSize = (SEGMENT_SIZE - HEADER_RECORD_SIZE) % recSize;
        attempt--;
    }
    Assert.assertNotNull(rec);
    int recordsToWrite = SEGMENT_SIZE / recSize;
    log.info("records to write " + recordsToWrite + " tail size " + (SEGMENT_SIZE - HEADER_RECORD_SIZE) % recSize);
    // Add more record for rollover to the next segment.
    recordsToWrite += 100;
    for (int i = 0; i < recordsToWrite; i++) walMgr.log(new MetastoreDataRecord(rec.key(), rec.value()));
    walMgr.flush(null, true);
    SegmentAware segmentAware = GridTestUtils.getFieldValue(walMgr, "segmentAware");
    // Await archiver move segment to WAL archive.
    waitForCondition(() -> segmentAware.lastArchivedAbsoluteIndex() == 0, 5_000);
    // Filling tail some garbage. Simulate tail garbage on rotate segment in WAL work directory.
    if (switchSegmentRecordSize > 1) {
        File seg = new File(workDir + ARCHIVE_SUB_DIR + "/0000000000000000.wal");
        FileIOFactory ioFactory = new RandomAccessFileIOFactory();
        FileIO seg0 = ioFactory.create(seg);
        byte[] bytes = new byte[tailSize];
        Random rnd = new Random();
        rnd.nextBytes(bytes);
        // Some record type.
        bytes[0] = (byte) (METASTORE_DATA_RECORD.ordinal() + 1);
        seg0.position((int) (seg0.size() - tailSize));
        seg0.write(bytes, 0, tailSize);
        seg0.force(true);
        seg0.close();
    }
    int expRecords = recordsToWrite;
    int actualRecords = 0;
    // Check that switch segment works as expected and all record is reachable.
    try (WALIterator it = walMgr.replay(null)) {
        while (it.hasNext()) {
            IgniteBiTuple<WALPointer, WALRecord> tup = it.next();
            WALRecord rec0 = tup.get2();
            if (rec0.type() == METASTORE_DATA_RECORD)
                actualRecords++;
        }
    }
    Assert.assertEquals("Not all records read during iteration.", expRecords, actualRecords);
}

Example 18 with FileIOFactory

use of org.apache.ignite.internal.processors.cache.persistence.file.FileIOFactory in project ignite by apache.

the class IgniteDataIntegrityTests method setUp.

/**
 */
@Before
public void setUp() throws Exception {
    File file = File.createTempFile("integrity", "dat");
    file.deleteOnExit();
    expBuf = new ByteBufferExpander(1024, ByteOrder.BIG_ENDIAN);
    FileIOFactory factory = new RandomAccessFileIOFactory();
    fileInput = new SimpleFileInput(factory.create(file), expBuf);
    ByteBuffer buf = ByteBuffer.allocate(1024);
    ThreadLocalRandom curr = ThreadLocalRandom.current();
    for (int i = 0; i < 1024; i += 16) {
        buf.putInt(curr.nextInt());
        buf.putInt(curr.nextInt());
        buf.putInt(curr.nextInt());
        buf.position(i);
        buf.putInt(FastCrc.calcCrc(buf, 12));
    }
    buf.rewind();
    fileInput.io().writeFully(buf);
    fileInput.io().force();
}

Example 19 with FileIOFactory

use of org.apache.ignite.internal.processors.cache.persistence.file.FileIOFactory in project ignite by apache.

the class DefragmentationMXBeanTest method testDefragmentationStatus.

/**
 * Test that JMX bean provides correct defragmentation status.
 * Description:
 * 1. Start one node,
 * 2. Put a load of data on it.
 * 3. Schedule defragmentation.
 * 4. Completely stop defragmentation when 128 partitions processed.
 * 5. Check defragmentation status.
 * 6. Continue defragmentation and wait for it to end.
 * 7. Check defragmentation finished.
 * @throws Exception If failed.
 */
@Test
public void testDefragmentationStatus() throws Exception {
    IgniteEx ig = startGrid(0);
    ig.cluster().state(ClusterState.ACTIVE);
    ig.getOrCreateCache(DEFAULT_CACHE_NAME + "1");
    IgniteCache<Object, Object> cache = ig.getOrCreateCache(DEFAULT_CACHE_NAME + "2");
    ig.getOrCreateCache(DEFAULT_CACHE_NAME + "3");
    for (int i = 0; i < 1024; i++) cache.put(i, i);
    forceCheckpoint(ig);
    DefragmentationMXBean mxBean = defragmentationMXBean(ig.name());
    mxBean.schedule("");
    stopGrid(0);
    blockCdl = new CountDownLatch(128);
    waitCdl = new CountDownLatch(1);
    UnaryOperator<IgniteConfiguration> cfgOp = cfg -> {
        DataStorageConfiguration dsCfg = cfg.getDataStorageConfiguration();
        FileIOFactory delegate = dsCfg.getFileIOFactory();
        dsCfg.setFileIOFactory((file, modes) -> {
            if (file.getName().contains("dfrg")) {
                if (blockCdl.getCount() == 0) {
                    try {
                        waitCdl.await();
                    } catch (InterruptedException ignore) {
                    // No-op.
                    }
                } else
                    blockCdl.countDown();
            }
            return delegate.create(file, modes);
        });
        return cfg;
    };
    IgniteInternalFuture<?> fut = GridTestUtils.runAsync(() -> {
        try {
            startGrid(0, cfgOp);
        } catch (Exception e) {
            // No-op.
            throw new RuntimeException(e);
        }
    });
    blockCdl.await();
    mxBean = defragmentationMXBean(ig.name());
    final IgniteKernal gridx = IgnitionEx.gridx(ig.name());
    final IgniteDefragmentation defragmentation = gridx.context().defragmentation();
    final IgniteDefragmentation.DefragmentationStatus status1 = defragmentation.status();
    assertEquals(status1.getStartTs(), mxBean.startTime());
    assertTrue(mxBean.inProgress());
    final int totalPartitions = status1.getTotalPartitions();
    assertEquals(totalPartitions, mxBean.totalPartitions());
    waitCdl.countDown();
    fut.get();
    ((GridCacheDatabaseSharedManager) grid(0).context().cache().context().database()).defragmentationManager().completionFuture().get();
    assertFalse(mxBean.inProgress());
    assertEquals(totalPartitions, mxBean.processedPartitions());
}

Example 20 with FileIOFactory

use of org.apache.ignite.internal.processors.cache.persistence.file.FileIOFactory in project ignite by apache.

the class IgniteClusterSnapshotRestoreMetricsTest method testRestoreSnapshotError.

/**
 * @throws Exception If fails.
 */
@Test
public void testRestoreSnapshotError() throws Exception {
    dfltCacheCfg.setCacheMode(CacheMode.REPLICATED);
    IgniteEx ignite = startGridsWithSnapshot(2, CACHE_KEYS_RANGE);
    String failingFilePath = Paths.get(FilePageStoreManager.cacheDirName(dfltCacheCfg), PART_FILE_PREFIX + (dfltCacheCfg.getAffinity().partitions() / 2) + FILE_SUFFIX).toString();
    FileIOFactory ioFactory = new RandomAccessFileIOFactory();
    String testErrMsg = "Test exception";
    ignite.context().cache().context().snapshotMgr().ioFactory((file, modes) -> {
        FileIO delegate = ioFactory.create(file, modes);
        if (file.getPath().endsWith(failingFilePath))
            throw new RuntimeException(testErrMsg);
        return delegate;
    });
    checkMetricsDefaults();
    ignite.snapshot().restoreSnapshot(SNAPSHOT_NAME, null);
    for (Ignite grid : G.allGrids()) {
        DynamicMBean mReg = metricRegistry(grid.name(), null, SNAPSHOT_RESTORE_METRICS);
        String nodeNameMsg = "node=" + grid.name();
        assertTrue(nodeNameMsg, GridTestUtils.waitForCondition(() -> getNumMetric("endTime", mReg) > 0, TIMEOUT));
        long startTime = getNumMetric("startTime", mReg);
        long endTime = getNumMetric("endTime", mReg);
        assertEquals(nodeNameMsg, SNAPSHOT_NAME, mReg.getAttribute("snapshotName"));
        assertFalse(nodeNameMsg, ((String) mReg.getAttribute("requestId")).isEmpty());
        assertTrue(nodeNameMsg, startTime > 0);
        assertTrue(nodeNameMsg, endTime >= startTime);
        assertTrue(nodeNameMsg, ((String) mReg.getAttribute("error")).contains(testErrMsg));
    }
}