Examples with ByteBufferExpander org.apache.ignite.internal.processors.cache.persistence.wal.ByteBufferExpander used on opensource projects

Example 1 with ByteBufferExpander

use of org.apache.ignite.internal.processors.cache.persistence.wal.ByteBufferExpander in project ignite by apache.

the class RecordV1Serializer method readSegmentHeader.

/**
 * Reads stored record from provided {@code io}.
 * NOTE: Method mutates position of {@code io}.
 *
 * @param io I/O interface for file.
 * @param segmentFileInputFactory File input factory.
 * @return Instance of {@link SegmentHeader} extracted from the file.
 * @throws IgniteCheckedException If failed to read serializer version.
 */
public static SegmentHeader readSegmentHeader(SegmentIO io, SegmentFileInputFactory segmentFileInputFactory) throws IgniteCheckedException, IOException {
    try (ByteBufferExpander buf = new ByteBufferExpander(HEADER_RECORD_SIZE, ByteOrder.nativeOrder())) {
        ByteBufferBackedDataInput in = segmentFileInputFactory.createFileInput(io, buf);
        in.ensure(HEADER_RECORD_SIZE);
        int recordType = in.readUnsignedByte();
        if (recordType == WALRecord.RecordType.STOP_ITERATION_RECORD_TYPE)
            throw new SegmentEofException("Reached logical end of the segment", null);
        WALRecord.RecordType type = WALRecord.RecordType.fromIndex(recordType - 1);
        if (type != WALRecord.RecordType.HEADER_RECORD)
            throw new IOException("Can't read serializer version", null);
        // Read file pointer.
        WALPointer ptr = readPosition(in);
        if (io.getSegmentId() != ptr.index())
            throw new SegmentEofException("Reached logical end of the segment by pointer", null);
        assert ptr.fileOffset() == 0 : "Header record should be placed at the beginning of file " + ptr;
        long hdrMagicNum = in.readLong();
        boolean compacted;
        if (hdrMagicNum == HeaderRecord.REGULAR_MAGIC)
            compacted = false;
        else if (hdrMagicNum == HeaderRecord.COMPACTED_MAGIC)
            compacted = true;
        else {
            throw new IOException("Magic is corrupted [exp=" + U.hexLong(HeaderRecord.REGULAR_MAGIC) + ", actual=" + U.hexLong(hdrMagicNum) + ']');
        }
        // Read serializer version.
        int ver = in.readInt();
        // Read and skip CRC.
        in.readInt();
        return new SegmentHeader(ver, compacted);
    }
}

Example 2 with ByteBufferExpander

use of org.apache.ignite.internal.processors.cache.persistence.wal.ByteBufferExpander in project ignite by apache.

the class IgniteWalIteratorFactory method readFileDescriptor.

/**
 * @param file File to read.
 * @param ioFactory IO factory.
 */
private FileDescriptor readFileDescriptor(File file, FileIOFactory ioFactory) {
    FileDescriptor ds = new FileDescriptor(file);
    try (SegmentIO fileIO = ds.toReadOnlyIO(ioFactory);
        ByteBufferExpander buf = new ByteBufferExpander(HEADER_RECORD_SIZE, ByteOrder.nativeOrder())) {
        final DataInput in = segmentFileInputFactory.createFileInput(fileIO, buf);
        // Header record must be agnostic to the serializer version.
        final int type = in.readUnsignedByte();
        if (type == RecordType.STOP_ITERATION_RECORD_TYPE) {
            if (log.isInfoEnabled())
                log.info("Reached logical end of the segment for file " + file);
            return null;
        }
        WALPointer ptr = readPosition(in);
        return new FileDescriptor(file, ptr.index());
    } catch (IOException e) {
        U.warn(log, "Failed to scan index from file [" + file + "]. Skipping this file during iteration", e);
        return null;
    }
}

Example 3 with ByteBufferExpander

use of org.apache.ignite.internal.processors.cache.persistence.wal.ByteBufferExpander in project ignite by apache.

the class IgniteDataIntegrityTests method testExpandBuffer.

/**
 */
@Test
public void testExpandBuffer() {
    ByteBufferExpander expBuf = new ByteBufferExpander(24, ByteOrder.nativeOrder());
    ByteBuffer b1 = expBuf.buffer();
    b1.put((byte) 1);
    b1.putInt(2);
    b1.putLong(3L);
    assertEquals(13, b1.position());
    assertEquals(24, b1.limit());
    ByteBuffer b2 = expBuf.expand(32);
    assertEquals(13, b2.position());
    assertEquals(24, b2.limit());
    b2.rewind();
    assertEquals(0, b2.position());
    assertEquals((byte) 1, b2.get());
    assertEquals(2, b2.getInt());
    assertEquals(3L, b2.getLong());
    assertEquals(13, b2.position());
    assertEquals(24, b2.limit());
    assertEquals(32, b2.capacity());
    b2.limit(b2.capacity());
    b2.putInt(4);
    b2.putInt(5);
    b2.putInt(6);
    assertEquals(25, b2.position());
    assertEquals(32, b2.limit());
    assertEquals(32, b2.capacity());
    b2.flip();
    assertEquals(0, b2.position());
    assertEquals((byte) 1, b2.get());
    assertEquals(2, b2.getInt());
    assertEquals(3L, b2.getLong());
    assertEquals(4, b2.getInt());
    assertEquals(5, b2.getInt());
    assertEquals(6, b2.getInt());
    assertEquals(25, b2.limit());
    assertEquals(32, b2.capacity());
}

Example 4 with ByteBufferExpander

use of org.apache.ignite.internal.processors.cache.persistence.wal.ByteBufferExpander in project ignite by apache.

the class StandaloneWalRecordsIterator method scanIndexesFromFileHeaders.

/**
 * This methods checks all provided files to be correct WAL segment.
 * Header record and its position is checked. WAL position is used to determine real index.
 * File index from file name is ignored.
 *
 * @param allFiles files to scan.
 * @return list of file descriptors with checked header records, having correct file index is set
 */
private List<FileWriteAheadLogManager.FileDescriptor> scanIndexesFromFileHeaders(@Nullable final File[] allFiles) {
    if (allFiles == null || allFiles.length == 0)
        return Collections.emptyList();
    final List<FileWriteAheadLogManager.FileDescriptor> resultingDescs = new ArrayList<>();
    for (File file : allFiles) {
        if (file.length() < HEADER_RECORD_SIZE)
            // filter out this segment as it is too short
            continue;
        FileWALPointer ptr;
        try (FileIO fileIO = ioFactory.create(file);
            ByteBufferExpander buf = new ByteBufferExpander(HEADER_RECORD_SIZE, ByteOrder.nativeOrder())) {
            final DataInput in = new FileInput(fileIO, buf);
            // Header record must be agnostic to the serializer version.
            final int type = in.readUnsignedByte();
            if (type == WALRecord.RecordType.STOP_ITERATION_RECORD_TYPE) {
                if (log.isInfoEnabled())
                    log.info("Reached logical end of the segment for file " + file);
                // filter out this segment
                continue;
            }
            ptr = RecordV1Serializer.readPosition(in);
        } catch (IOException e) {
            U.warn(log, "Failed to scan index from file [" + file + "]. Skipping this file during iteration", e);
            // filter out this segment
            continue;
        }
        resultingDescs.add(new FileWriteAheadLogManager.FileDescriptor(file, ptr.index()));
    }
    Collections.sort(resultingDescs);
    return resultingDescs;
}

Example 5 with ByteBufferExpander

use of org.apache.ignite.internal.processors.cache.persistence.wal.ByteBufferExpander in project ignite by apache.

the class IgniteDataIntegrityTests method setUp.

/**
 */
@Before
public void setUp() throws Exception {
    File file = File.createTempFile("integrity", "dat");
    file.deleteOnExit();
    expBuf = new ByteBufferExpander(1024, ByteOrder.BIG_ENDIAN);
    FileIOFactory factory = new RandomAccessFileIOFactory();
    fileInput = new SimpleFileInput(factory.create(file), expBuf);
    ByteBuffer buf = ByteBuffer.allocate(1024);
    ThreadLocalRandom curr = ThreadLocalRandom.current();
    for (int i = 0; i < 1024; i += 16) {
        buf.putInt(curr.nextInt());
        buf.putInt(curr.nextInt());
        buf.putInt(curr.nextInt());
        buf.position(i);
        buf.putInt(FastCrc.calcCrc(buf, 12));
    }
    buf.rewind();
    fileInput.io().writeFully(buf);
    fileInput.io().force();
}