Search in sources :

Example 31 with Catalog

use of io.aeron.archive.Catalog in project aeron by real-logic.

the class ArchiveToolTests method verifyChecksum.

@Test
void verifyChecksum() {
    final Checksum checksum = crc32();
    try (Catalog catalog = openCatalogReadWrite(archiveDir, epochClock, MIN_CAPACITY, checksum, null)) {
        assertRecording(catalog, validRecording51, (recordingDescriptorOffset, headerEncoder, headerDecoder, descriptorEncoder, descriptorDecoder) -> catalog.updateChecksum(recordingDescriptorOffset));
        assertRecording(catalog, validRecording6, (recordingDescriptorOffset, headerEncoder, headerDecoder, descriptorEncoder, descriptorDecoder) -> catalog.updateChecksum(recordingDescriptorOffset));
    }
    assertFalse(verify(out, archiveDir, of(APPLY_CHECKSUM), checksum, epochClock, (file) -> true));
    try (Catalog catalog = openCatalogReadOnly(archiveDir, epochClock)) {
        assertRecording(catalog, validRecording0, INVALID, 0, 0, NULL_POSITION, 15, NULL_TIMESTAMP, 0, 2, "ch2", "src2");
        assertRecording(catalog, validRecording1, INVALID, 0, 1024, NULL_POSITION, 16, NULL_TIMESTAMP, 0, 2, "ch2", "src2");
        assertRecording(catalog, validRecording2, INVALID, 0, TERM_LENGTH * 3 + 96, NULL_POSITION, 17, NULL_TIMESTAMP, 0, 2, "ch2", "src2");
        assertRecording(catalog, validRecording3, INVALID, 0, 7 * TERM_LENGTH + 96, 7 * TERM_LENGTH + 128, 18, NULL_TIMESTAMP, 7, 13, "ch2", "src2");
        assertRecording(catalog, validRecording4, INVALID, 0, 21 * TERM_LENGTH + (TERM_LENGTH - 64), 22 * TERM_LENGTH + 992, 19, 1, -25, 7, "ch2", "src2");
        assertRecording(catalog, validRecording51, VALID, -940881948, 0, 64 + PAGE_SIZE, 20, 777, 0, 20, "ch2", "src2");
        assertRecording(catalog, validRecording52, INVALID, 0, 0, NULL_POSITION, 21, NULL_TIMESTAMP, 0, 52, "ch2", "src2");
        assertRecording(catalog, validRecording53, INVALID, 0, 0, NULL_POSITION, 22, NULL_TIMESTAMP, 0, 53, "ch2", "src2");
        assertRecording(catalog, validRecording6, VALID, -175549265, 352, 960, 23, 100, 0, 6, "ch2", "src2");
    }
    Mockito.verify(out, times(24)).println(any(String.class));
}
Also used : ArgumentMatchers.any(org.mockito.ArgumentMatchers.any) Arrays(java.util.Arrays) FRAME_ALIGNMENT(io.aeron.logbuffer.FrameDescriptor.FRAME_ALIGNMENT) RECORDING_SEGMENT_SUFFIX(io.aeron.archive.Archive.Configuration.RECORDING_SEGMENT_SUFFIX) Catalog(io.aeron.archive.Catalog) IoUtil(org.agrona.IoUtil) ByteBuffer(java.nio.ByteBuffer) ArrayList(java.util.ArrayList) READ(java.nio.file.StandardOpenOption.READ) ByteBuffer.allocate(java.nio.ByteBuffer.allocate) Files.createTempDirectory(java.nio.file.Files.createTempDirectory) EnumSet.allOf(java.util.EnumSet.allOf) AeronArchive(io.aeron.archive.client.AeronArchive) Path(java.nio.file.Path) WRITE(java.nio.file.StandardOpenOption.WRITE) MethodSource(org.junit.jupiter.params.provider.MethodSource) PrintStream(java.io.PrintStream) MutableBoolean(org.agrona.collections.MutableBoolean) LogBufferDescriptor.positionBitsToShift(io.aeron.logbuffer.LogBufferDescriptor.positionBitsToShift) ArchiveTool(io.aeron.archive.ArchiveTool) Collections.emptySet(java.util.Collections.emptySet) VERIFY_ALL_SEGMENT_FILES(io.aeron.archive.ArchiveTool.VerifyOption.VERIFY_ALL_SEGMENT_FILES) IOException(java.io.IOException) Mockito.times(org.mockito.Mockito.times) Checksum(io.aeron.archive.checksum.Checksum) Arguments(org.junit.jupiter.params.provider.Arguments) File(java.io.File) Mockito(org.mockito.Mockito) INVALID(io.aeron.archive.codecs.RecordingState.INVALID) VALID(io.aeron.archive.codecs.RecordingState.VALID) ParameterizedTest(org.junit.jupiter.params.ParameterizedTest) List(java.util.List) Collections.addAll(java.util.Collections.addAll) org.junit.jupiter.api(org.junit.jupiter.api) Executable(org.junit.jupiter.api.function.Executable) APPLY_CHECKSUM(io.aeron.archive.ArchiveTool.VerifyOption.APPLY_CHECKSUM) EnumSet.of(java.util.EnumSet.of) Assertions(org.junit.jupiter.api.Assertions) RecordingState(io.aeron.archive.codecs.RecordingState) EpochClock(org.agrona.concurrent.EpochClock) DataHeaderFlyweight(io.aeron.protocol.DataHeaderFlyweight) Checksums.crc32(io.aeron.archive.checksum.Checksums.crc32) Archive.segmentFileName(io.aeron.archive.Archive.segmentFileName) FileChannel(java.nio.channels.FileChannel) Files.deleteIfExists(java.nio.file.Files.deleteIfExists) LogBufferDescriptor.computeTermIdFromPosition(io.aeron.logbuffer.LogBufferDescriptor.computeTermIdFromPosition) Checksum(io.aeron.archive.checksum.Checksum) Catalog(io.aeron.archive.Catalog) ParameterizedTest(org.junit.jupiter.params.ParameterizedTest)

Example 32 with Catalog

use of io.aeron.archive.Catalog in project aeron by real-logic.

the class ArchiveToolTests method before.

@SuppressWarnings("MethodLength")
@BeforeEach
void before() throws IOException {
    archiveDir = ArchiveTests.makeTestDirectory();
    try (Catalog catalog = new Catalog(archiveDir, null, 0, 1024, epochClock, null, null)) {
        invalidRecording0 = catalog.addNewRecording(NULL_POSITION, NULL_POSITION, 1, NULL_TIMESTAMP, 0, SEGMENT_LENGTH, TERM_LENGTH, MTU_LENGTH, 1, 1, "ch1", "ch1?tag=ERR", "src1");
        invalidRecording1 = catalog.addNewRecording(FRAME_ALIGNMENT - 7, NULL_POSITION, 2, NULL_TIMESTAMP, 0, SEGMENT_LENGTH, TERM_LENGTH, MTU_LENGTH, 1, 1, "ch1", "ch1?tag=ERR", "src1");
        invalidRecording2 = catalog.addNewRecording(1024, FRAME_ALIGNMENT * 2, 3, NULL_TIMESTAMP, 0, SEGMENT_LENGTH, TERM_LENGTH, MTU_LENGTH, 1, 1, "ch1", "ch1?tag=ERR", "src1");
        invalidRecording3 = catalog.addNewRecording(0, FRAME_ALIGNMENT * 5 + 11, 4, NULL_TIMESTAMP, 0, SEGMENT_LENGTH, TERM_LENGTH, MTU_LENGTH, 1, 1, "ch1", "ch1?tag=ERR", "src1");
        invalidRecording4 = catalog.addNewRecording(SEGMENT_LENGTH, NULL_POSITION, 5, NULL_TIMESTAMP, 0, SEGMENT_LENGTH, TERM_LENGTH, MTU_LENGTH, 1, 1, "ch1", "ch1?tag=ERR", "src1");
        invalidRecording5 = catalog.addNewRecording(0, SEGMENT_LENGTH, 6, NULL_TIMESTAMP, 0, SEGMENT_LENGTH, TERM_LENGTH, MTU_LENGTH, 1, 1, "ch1", "ch1?tag=ERR", "src1");
        invalidRecording6 = catalog.addNewRecording(0, NULL_POSITION, 7, NULL_TIMESTAMP, 0, SEGMENT_LENGTH, TERM_LENGTH, MTU_LENGTH, 1, 1, "ch1", "ch1?tag=ERR", "src1");
        invalidRecording7 = catalog.addNewRecording(0, NULL_POSITION, 8, NULL_TIMESTAMP, 0, SEGMENT_LENGTH, TERM_LENGTH, MTU_LENGTH, 1, 1, "ch1", "ch1?tag=ERR", "src1");
        invalidRecording8 = catalog.addNewRecording(0, NULL_POSITION, 9, NULL_TIMESTAMP, 0, SEGMENT_LENGTH, TERM_LENGTH, MTU_LENGTH, 1, 1, "ch1", "ch1?tag=ERR", "src1");
        invalidRecording9 = catalog.addNewRecording(0, NULL_POSITION, 10, NULL_TIMESTAMP, 0, SEGMENT_LENGTH, TERM_LENGTH, MTU_LENGTH, 1, 1, "ch1", "ch1?tag=ERR", "src1");
        invalidRecording10 = catalog.addNewRecording(128, NULL_POSITION, 11, NULL_TIMESTAMP, 0, SEGMENT_LENGTH, TERM_LENGTH, MTU_LENGTH, 1, 1, "ch1", "ch1?tag=ERR", "src1");
        invalidRecording11 = catalog.addNewRecording(0, NULL_POSITION, 12, NULL_TIMESTAMP, 5, SEGMENT_LENGTH, TERM_LENGTH, MTU_LENGTH, 1, 1, "ch1", "ch1?tag=ERR", "src1");
        invalidRecording12 = catalog.addNewRecording(0, NULL_POSITION, 13, NULL_TIMESTAMP, 9, SEGMENT_LENGTH, TERM_LENGTH, MTU_LENGTH, 1, 6, "ch1", "ch1?tag=ERR", "src1");
        invalidRecording13 = catalog.addNewRecording(0, NULL_POSITION, 14, NULL_TIMESTAMP, 0, SEGMENT_LENGTH, TERM_LENGTH, MTU_LENGTH, 1, 13, "ch1", "ch1?tag=ERR", "src1");
        invalidRecording14 = catalog.addNewRecording(128, NULL_POSITION, -14, 41, -14, SEGMENT_LENGTH, TERM_LENGTH, MTU_LENGTH, 1, 0, "ch1", "ch1?tag=ERR", "src1");
        validRecording0 = catalog.addNewRecording(0, NULL_POSITION, 15, NULL_TIMESTAMP, 0, SEGMENT_LENGTH, TERM_LENGTH, MTU_LENGTH, 2, 2, "ch2", "ch2?tag=OK", "src2");
        validRecording1 = catalog.addNewRecording(1024, NULL_POSITION, 16, NULL_TIMESTAMP, 0, SEGMENT_LENGTH, TERM_LENGTH, MTU_LENGTH, 2, 2, "ch2", "ch2?tag=OK", "src2");
        validRecording2 = catalog.addNewRecording(3 * TERM_LENGTH + 96, NULL_POSITION, 17, NULL_TIMESTAMP, 0, SEGMENT_LENGTH, TERM_LENGTH, MTU_LENGTH, 2, 2, "ch2", "ch2?tag=OK", "src2");
        validRecording3 = catalog.addNewRecording(7 * TERM_LENGTH + 96, 7 * TERM_LENGTH + 128, 18, NULL_TIMESTAMP, 7, SEGMENT_LENGTH, TERM_LENGTH, MTU_LENGTH, 999, 13, "ch2", "ch2?tag=OK", "src2");
        validRecording4 = catalog.addNewRecording(21 * TERM_LENGTH + (TERM_LENGTH - 64), 22 * TERM_LENGTH + 992, 19, 1, -25, SEGMENT_LENGTH, TERM_LENGTH, MTU_LENGTH, -999, 7, "ch2", "ch2?tag=OK", "src2");
        validRecording51 = catalog.addNewRecording(0, 64 + PAGE_SIZE, 20, 777, 0, SEGMENT_LENGTH, TERM_LENGTH, MTU_LENGTH, -1, 20, "ch2", "ch2?tag=OK", "src2");
        validRecording52 = catalog.addNewRecording(0, NULL_POSITION, 21, NULL_TIMESTAMP, 0, SEGMENT_LENGTH, TERM_LENGTH, MTU_LENGTH, 52, 52, "ch2", "ch2?tag=OK", "src2");
        validRecording53 = catalog.addNewRecording(0, NULL_POSITION, 22, NULL_TIMESTAMP, 0, SEGMENT_LENGTH, TERM_LENGTH, MTU_LENGTH, 53, 53, "ch2", "ch2?tag=OK", "src2");
        validRecording6 = catalog.addNewRecording(352, NULL_POSITION, 23, NULL_TIMESTAMP, 0, SEGMENT_LENGTH, TERM_LENGTH, MTU_LENGTH, 10, 6, "ch2", "ch2?tag=OK", "src2");
    }
    writeToSegmentFile(createFile(segmentFileName(invalidRecording4, 0)), SEGMENT_LENGTH, (byteBuffer, dataHeaderFlyweight, fileChannel) -> fileChannel.write(byteBuffer));
    writeToSegmentFile(createFile(segmentFileName(invalidRecording5, 0)), SEGMENT_LENGTH, (byteBuffer, dataHeaderFlyweight, fileChannel) -> fileChannel.write(byteBuffer));
    createFile(invalidRecording6 + "-" + RECORDING_SEGMENT_SUFFIX);
    createFile(invalidRecording7 + "-this-will-not-parse" + RECORDING_SEGMENT_SUFFIX);
    createFile(segmentFileName(invalidRecording8, -1024));
    createDirectory(segmentFileName(invalidRecording9, 0));
    writeToSegmentFile(createFile(segmentFileName(invalidRecording10, 0)), 150, (byteBuffer, dataHeaderFlyweight, fileChannel) -> fileChannel.write(byteBuffer));
    writeToSegmentFile(createFile(segmentFileName(invalidRecording11, 0)), SEGMENT_LENGTH, (byteBuffer, dataHeaderFlyweight, fileChannel) -> {
        dataHeaderFlyweight.headerType(HDR_TYPE_DATA);
        dataHeaderFlyweight.frameLength(64);
        dataHeaderFlyweight.streamId(1);
        dataHeaderFlyweight.termOffset(0);
        dataHeaderFlyweight.termId(5);
        fileChannel.write(byteBuffer);
        byteBuffer.clear();
        dataHeaderFlyweight.headerType(HDR_TYPE_DATA);
        dataHeaderFlyweight.frameLength(40);
        dataHeaderFlyweight.termId(-1);
        fileChannel.write(byteBuffer, 64);
    });
    writeToSegmentFile(createFile(segmentFileName(invalidRecording12, 0)), SEGMENT_LENGTH, (byteBuffer, dataHeaderFlyweight, fileChannel) -> {
        dataHeaderFlyweight.headerType(HDR_TYPE_DATA);
        dataHeaderFlyweight.frameLength(64);
        dataHeaderFlyweight.streamId(6);
        dataHeaderFlyweight.termOffset(0);
        dataHeaderFlyweight.termId(9);
        fileChannel.write(byteBuffer);
        byteBuffer.clear();
        dataHeaderFlyweight.headerType(HDR_TYPE_DATA);
        dataHeaderFlyweight.frameLength(50);
        dataHeaderFlyweight.termOffset(-1);
        fileChannel.write(byteBuffer, 64);
    });
    writeToSegmentFile(createFile(segmentFileName(invalidRecording13, 0)), SEGMENT_LENGTH, (byteBuffer, dataHeaderFlyweight, fileChannel) -> {
        dataHeaderFlyweight.headerType(HDR_TYPE_DATA);
        dataHeaderFlyweight.frameLength(64);
        dataHeaderFlyweight.streamId(13);
        dataHeaderFlyweight.termOffset(0);
        dataHeaderFlyweight.termId(0);
        fileChannel.write(byteBuffer);
        byteBuffer.clear();
        dataHeaderFlyweight.headerType(HDR_TYPE_DATA);
        dataHeaderFlyweight.frameLength(60);
        dataHeaderFlyweight.streamId(-1);
        fileChannel.write(byteBuffer, 64);
    });
    writeToSegmentFile(createFile(segmentFileName(invalidRecording14, 0)), 228, (byteBuffer, dataHeaderFlyweight, fileChannel) -> {
        dataHeaderFlyweight.headerType(HDR_TYPE_DATA);
        dataHeaderFlyweight.frameLength(100);
        dataHeaderFlyweight.streamId(0);
        dataHeaderFlyweight.termOffset(128);
        dataHeaderFlyweight.termId(-14);
        fileChannel.write(byteBuffer, 128);
    });
    writeToSegmentFile(createFile(segmentFileName(validRecording0, 0)), SEGMENT_LENGTH, (byteBuffer, dataHeaderFlyweight, fileChannel) -> {
        for (int i = 0, termOffset = 0; i < TERM_LENGTH / MTU_LENGTH - 1; i++, termOffset += MTU_LENGTH) {
            byteBuffer.clear().limit(MTU_LENGTH);
            dataHeaderFlyweight.headerType(HDR_TYPE_DATA);
            dataHeaderFlyweight.streamId(2);
            dataHeaderFlyweight.frameLength(MTU_LENGTH);
            dataHeaderFlyweight.sessionId(876814387);
            dataHeaderFlyweight.termOffset(termOffset);
            fileChannel.write(byteBuffer, termOffset);
        }
        byteBuffer.clear().limit(256);
        dataHeaderFlyweight.frameLength(256);
        dataHeaderFlyweight.termOffset(TERM_LENGTH - MTU_LENGTH);
        dataHeaderFlyweight.sessionId(1025596259);
        fileChannel.write(byteBuffer, TERM_LENGTH - MTU_LENGTH);
        byteBuffer.clear().limit(MTU_LENGTH - 256);
        dataHeaderFlyweight.headerType(HDR_TYPE_PAD);
        // should not be checked
        dataHeaderFlyweight.sessionId(-1);
        dataHeaderFlyweight.frameLength(MTU_LENGTH - 256);
        dataHeaderFlyweight.termOffset(TERM_LENGTH - MTU_LENGTH + 256);
        fileChannel.write(byteBuffer, TERM_LENGTH - MTU_LENGTH + 256);
        byteBuffer.clear().limit(64);
        dataHeaderFlyweight.headerType(HDR_TYPE_DATA);
        dataHeaderFlyweight.frameLength(64);
        dataHeaderFlyweight.termOffset(0);
        dataHeaderFlyweight.termId(1);
        dataHeaderFlyweight.sessionId(420107693);
        fileChannel.write(byteBuffer, TERM_LENGTH);
    });
    writeToSegmentFile(createFile(segmentFileName(validRecording2, 3 * TERM_LENGTH)), SEGMENT_LENGTH, (byteBuffer, dataHeaderFlyweight, fileChannel) -> {
        dataHeaderFlyweight.headerType(HDR_TYPE_DATA);
        dataHeaderFlyweight.frameLength(0);
        fileChannel.write(byteBuffer, 96);
    });
    writeToSegmentFile(createFile(segmentFileName(validRecording3, 7 * TERM_LENGTH)), SEGMENT_LENGTH, (byteBuffer, dataHeaderFlyweight, fileChannel) -> {
        int fileOffset = 96;
        dataHeaderFlyweight.headerType(HDR_TYPE_DATA);
        dataHeaderFlyweight.frameLength(128);
        dataHeaderFlyweight.streamId(13);
        dataHeaderFlyweight.termId(14);
        dataHeaderFlyweight.termOffset(96);
        dataHeaderFlyweight.setMemory(HEADER_LENGTH, 128 - HEADER_LENGTH, (byte) 13);
        fileChannel.write(byteBuffer, fileOffset);
        fileOffset += 128;
        final int segmentFileBasePosition = 7 * TERM_LENGTH;
        final int positionBitsToShift = positionBitsToShift(TERM_LENGTH);
        for (int i = 0; i < (SEGMENT_LENGTH / MTU_LENGTH) - 1; i++) {
            byteBuffer.clear();
            final int termId = computeTermIdFromPosition(segmentFileBasePosition + fileOffset, positionBitsToShift, 7);
            dataHeaderFlyweight.frameLength(MTU_LENGTH);
            dataHeaderFlyweight.termId(termId);
            dataHeaderFlyweight.termOffset(fileOffset & (TERM_LENGTH - 1));
            dataHeaderFlyweight.setMemory(HEADER_LENGTH, MTU_LENGTH - HEADER_LENGTH, (byte) i);
            fileChannel.write(byteBuffer, fileOffset);
            fileOffset += MTU_LENGTH;
        }
        final int lastFrameLength = MTU_LENGTH - 224;
        assertEquals(SEGMENT_LENGTH - lastFrameLength, fileOffset);
        byteBuffer.clear();
        dataHeaderFlyweight.frameLength(lastFrameLength);
        dataHeaderFlyweight.termOffset(fileOffset & (TERM_LENGTH - 1));
        for (int i = 0; i < lastFrameLength - HEADER_LENGTH; i++) {
            dataHeaderFlyweight.setMemory(HEADER_LENGTH + i, 1, (byte) i);
        }
        dataHeaderFlyweight.setMemory(HEADER_LENGTH, 1, (byte) -128);
        dataHeaderFlyweight.setMemory(lastFrameLength - 1, 1, (byte) 127);
        fileChannel.write(byteBuffer, fileOffset);
    });
    writeToSegmentFile(createFile(segmentFileName(validRecording3, 11 * TERM_LENGTH)), SEGMENT_LENGTH, (byteBuffer, dataHeaderFlyweight, fileChannel) -> {
        dataHeaderFlyweight.headerType(HDR_TYPE_DATA);
        dataHeaderFlyweight.frameLength(180);
        dataHeaderFlyweight.streamId(13);
        dataHeaderFlyweight.termId(18);
        dataHeaderFlyweight.termOffset(0);
        fileChannel.write(byteBuffer);
        byteBuffer.clear();
        dataHeaderFlyweight.frameLength(128);
        dataHeaderFlyweight.termId(18);
        dataHeaderFlyweight.termOffset(192);
        fileChannel.write(byteBuffer, 192);
    });
    writeToSegmentFile(createFile(segmentFileName(validRecording4, 0)), SEGMENT_LENGTH, (byteBuffer, dataHeaderFlyweight, fileChannel) -> {
        dataHeaderFlyweight.headerType(HDR_TYPE_DATA);
        dataHeaderFlyweight.frameLength(64);
        dataHeaderFlyweight.streamId(Integer.MAX_VALUE);
        fileChannel.write(byteBuffer);
    });
    writeToSegmentFile(createFile(segmentFileName(validRecording4, 21 * TERM_LENGTH)), SEGMENT_LENGTH, (byteBuffer, dataHeaderFlyweight, fileChannel) -> {
        dataHeaderFlyweight.headerType(HDR_TYPE_PAD);
        dataHeaderFlyweight.frameLength(64);
        dataHeaderFlyweight.streamId(7);
        dataHeaderFlyweight.termId(-4);
        dataHeaderFlyweight.termOffset(TERM_LENGTH - 64);
        fileChannel.write(byteBuffer, TERM_LENGTH - 64);
        byteBuffer.clear();
        dataHeaderFlyweight.frameLength(992);
        dataHeaderFlyweight.termId(-3);
        dataHeaderFlyweight.termOffset(0);
        fileChannel.write(byteBuffer, TERM_LENGTH);
    });
    // Page straddle: valid checksum
    writeToSegmentFile(createFile(segmentFileName(validRecording51, 0)), SEGMENT_LENGTH, (byteBuffer, dataHeaderFlyweight, fileChannel) -> {
        dataHeaderFlyweight.headerType(HDR_TYPE_DATA);
        dataHeaderFlyweight.frameLength(64);
        dataHeaderFlyweight.streamId(20);
        dataHeaderFlyweight.sessionId(420107693);
        fileChannel.write(byteBuffer);
        byteBuffer.clear();
        dataHeaderFlyweight.frameLength(PAGE_SIZE);
        dataHeaderFlyweight.termOffset(64);
        dataHeaderFlyweight.sessionId(2057703623);
        fileChannel.write(byteBuffer, dataHeaderFlyweight.termOffset());
    });
    // Page straddle: non-zero bytes in every page since the straddle
    writeToSegmentFile(createFile(segmentFileName(validRecording52, 0)), SEGMENT_LENGTH, (byteBuffer, dataHeaderFlyweight, fileChannel) -> {
        dataHeaderFlyweight.headerType(HDR_TYPE_DATA);
        dataHeaderFlyweight.frameLength(111);
        dataHeaderFlyweight.streamId(52);
        fileChannel.write(byteBuffer);
        byteBuffer.clear();
        dataHeaderFlyweight.frameLength(MTU_LENGTH);
        dataHeaderFlyweight.termOffset(128);
        dataHeaderFlyweight.putBytes(PAGE_SIZE + 1, new byte[] { 0, 0, -1, 0, 7 });
        dataHeaderFlyweight.putByte(2 * PAGE_SIZE + 512, (byte) 127);
        dataHeaderFlyweight.putByte(MTU_LENGTH - 128 - 1, (byte) 1);
        dataHeaderFlyweight.putByte(MTU_LENGTH - 1, (byte) -128);
        fileChannel.write(byteBuffer, dataHeaderFlyweight.termOffset());
    });
    // Page straddle: all zeroes in one of the pages after the straddle
    writeToSegmentFile(createFile(segmentFileName(validRecording53, 0)), SEGMENT_LENGTH, (byteBuffer, dataHeaderFlyweight, fileChannel) -> {
        dataHeaderFlyweight.headerType(HDR_TYPE_DATA);
        dataHeaderFlyweight.frameLength(60);
        dataHeaderFlyweight.streamId(53);
        fileChannel.write(byteBuffer);
        byteBuffer.clear();
        dataHeaderFlyweight.frameLength(3 * PAGE_SIZE);
        dataHeaderFlyweight.termOffset(64);
        dataHeaderFlyweight.setMemory(PAGE_SIZE - 64, PAGE_SIZE, (byte) 111);
        dataHeaderFlyweight.setMemory(3 * PAGE_SIZE - 64, 64, (byte) -128);
        fileChannel.write(byteBuffer, dataHeaderFlyweight.termOffset());
    });
    writeToSegmentFile(createFile(segmentFileName(validRecording6, 0)), SEGMENT_LENGTH, (byteBuffer, dataHeaderFlyweight, fileChannel) -> {
        dataHeaderFlyweight.headerType(HDR_TYPE_DATA);
        dataHeaderFlyweight.frameLength(64);
        dataHeaderFlyweight.streamId(6);
        dataHeaderFlyweight.termOffset(352);
        // CRC32
        dataHeaderFlyweight.sessionId(-1960800604);
        dataHeaderFlyweight.setMemory(HEADER_LENGTH, 20, (byte) 1);
        fileChannel.write(byteBuffer, 352);
        byteBuffer.clear();
        dataHeaderFlyweight.frameLength(544);
        dataHeaderFlyweight.termOffset(416);
        // CRC32
        dataHeaderFlyweight.sessionId(-327206874);
        dataHeaderFlyweight.setMemory(HEADER_LENGTH, 256, (byte) 11);
        dataHeaderFlyweight.setMemory(HEADER_LENGTH + 256, 256, (byte) -20);
        fileChannel.write(byteBuffer, 416);
    });
}
Also used : Catalog(io.aeron.archive.Catalog)

Example 33 with Catalog

use of io.aeron.archive.Catalog in project aeron by real-logic.

the class ArchiveToolTests method capacityReturnsCurrentCapacityInBytesOfTheCatalog.

@Test
void capacityReturnsCurrentCapacityInBytesOfTheCatalog() {
    final long capacity;
    try (Catalog catalog = openCatalogReadOnly(archiveDir, epochClock)) {
        capacity = catalog.capacity();
    }
    assertEquals(capacity, capacity(archiveDir));
}
Also used : Catalog(io.aeron.archive.Catalog) ParameterizedTest(org.junit.jupiter.params.ParameterizedTest)

Example 34 with Catalog

use of io.aeron.archive.Catalog in project aeron by real-logic.

the class ArchiveToolTests method verifyRecordingValidRecordingPerformCRC.

@Test
void verifyRecordingValidRecordingPerformCRC() {
    final Checksum checksum = crc32();
    try (Catalog catalog = openCatalogReadWrite(archiveDir, epochClock, MIN_CAPACITY, checksum, null)) {
        assertRecording(catalog, validRecording6, (recordingDescriptorOffset, headerEncoder, headerDecoder, descriptorEncoder, descriptorDecoder) -> catalog.updateChecksum(recordingDescriptorOffset));
    }
    assertTrue(verifyRecording(out, archiveDir, validRecording6, of(APPLY_CHECKSUM), checksum, epochClock, (file) -> false));
    try (Catalog catalog = openCatalogReadOnly(archiveDir, epochClock)) {
        assertRecording(catalog, validRecording6, VALID, -175549265, 352, 960, 23, 100, 0, 6, "ch2", "src2");
    }
}
Also used : ArgumentMatchers.any(org.mockito.ArgumentMatchers.any) Arrays(java.util.Arrays) FRAME_ALIGNMENT(io.aeron.logbuffer.FrameDescriptor.FRAME_ALIGNMENT) RECORDING_SEGMENT_SUFFIX(io.aeron.archive.Archive.Configuration.RECORDING_SEGMENT_SUFFIX) Catalog(io.aeron.archive.Catalog) IoUtil(org.agrona.IoUtil) ByteBuffer(java.nio.ByteBuffer) ArrayList(java.util.ArrayList) READ(java.nio.file.StandardOpenOption.READ) ByteBuffer.allocate(java.nio.ByteBuffer.allocate) Files.createTempDirectory(java.nio.file.Files.createTempDirectory) EnumSet.allOf(java.util.EnumSet.allOf) AeronArchive(io.aeron.archive.client.AeronArchive) Path(java.nio.file.Path) WRITE(java.nio.file.StandardOpenOption.WRITE) MethodSource(org.junit.jupiter.params.provider.MethodSource) PrintStream(java.io.PrintStream) MutableBoolean(org.agrona.collections.MutableBoolean) LogBufferDescriptor.positionBitsToShift(io.aeron.logbuffer.LogBufferDescriptor.positionBitsToShift) ArchiveTool(io.aeron.archive.ArchiveTool) Collections.emptySet(java.util.Collections.emptySet) VERIFY_ALL_SEGMENT_FILES(io.aeron.archive.ArchiveTool.VerifyOption.VERIFY_ALL_SEGMENT_FILES) IOException(java.io.IOException) Mockito.times(org.mockito.Mockito.times) Checksum(io.aeron.archive.checksum.Checksum) Arguments(org.junit.jupiter.params.provider.Arguments) File(java.io.File) Mockito(org.mockito.Mockito) INVALID(io.aeron.archive.codecs.RecordingState.INVALID) VALID(io.aeron.archive.codecs.RecordingState.VALID) ParameterizedTest(org.junit.jupiter.params.ParameterizedTest) List(java.util.List) Collections.addAll(java.util.Collections.addAll) org.junit.jupiter.api(org.junit.jupiter.api) Executable(org.junit.jupiter.api.function.Executable) APPLY_CHECKSUM(io.aeron.archive.ArchiveTool.VerifyOption.APPLY_CHECKSUM) EnumSet.of(java.util.EnumSet.of) Assertions(org.junit.jupiter.api.Assertions) RecordingState(io.aeron.archive.codecs.RecordingState) EpochClock(org.agrona.concurrent.EpochClock) DataHeaderFlyweight(io.aeron.protocol.DataHeaderFlyweight) Checksums.crc32(io.aeron.archive.checksum.Checksums.crc32) Archive.segmentFileName(io.aeron.archive.Archive.segmentFileName) FileChannel(java.nio.channels.FileChannel) Files.deleteIfExists(java.nio.file.Files.deleteIfExists) LogBufferDescriptor.computeTermIdFromPosition(io.aeron.logbuffer.LogBufferDescriptor.computeTermIdFromPosition) Checksum(io.aeron.archive.checksum.Checksum) Catalog(io.aeron.archive.Catalog) ParameterizedTest(org.junit.jupiter.params.ParameterizedTest)

Example 35 with Catalog

use of io.aeron.archive.Catalog in project aeron by real-logic.

the class ArchiveToolTests method checksumLastSegmentFile.

@Test
void checksumLastSegmentFile() {
    checksum(out, archiveDir, false, crc32(), epochClock);
    assertFalse(verify(out, archiveDir, allOf(VerifyOption.class), crc32(), epochClock, (file) -> false));
    try (Catalog catalog = openCatalogReadOnly(archiveDir, epochClock)) {
        assertRecording(catalog, validRecording0, VALID, 356725588, 0, TERM_LENGTH + 64, 15, 100, 0, 2, "ch2", "src2");
        assertRecording(catalog, validRecording1, VALID, -1571032591, 1024, 1024, 16, 200, 0, 2, "ch2", "src2");
        assertRecording(catalog, validRecording2, VALID, 114203747, TERM_LENGTH * 3 + 96, TERM_LENGTH * 3 + 96, 17, 300, 0, 2, "ch2", "src2");
        assertRecording(catalog, validRecording3, INVALID, 963969455, 7 * TERM_LENGTH + 96, 7 * TERM_LENGTH + 128, 18, NULL_TIMESTAMP, 7, 13, "ch2", "src2");
        assertRecording(catalog, validRecording4, INVALID, 162247708, 21 * TERM_LENGTH + (TERM_LENGTH - 64), 22 * TERM_LENGTH + 992, 19, 1, -25, 7, "ch2", "src2");
        assertRecording(catalog, validRecording51, VALID, -940881948, 0, 64 + PAGE_SIZE, 20, 777, 0, 20, "ch2", "src2");
        assertRecording(catalog, validRecording6, VALID, -175549265, 352, 960, 23, 600, 0, 6, "ch2", "src2");
    }
}
Also used : ArgumentMatchers.any(org.mockito.ArgumentMatchers.any) Arrays(java.util.Arrays) FRAME_ALIGNMENT(io.aeron.logbuffer.FrameDescriptor.FRAME_ALIGNMENT) RECORDING_SEGMENT_SUFFIX(io.aeron.archive.Archive.Configuration.RECORDING_SEGMENT_SUFFIX) Catalog(io.aeron.archive.Catalog) IoUtil(org.agrona.IoUtil) ByteBuffer(java.nio.ByteBuffer) ArrayList(java.util.ArrayList) READ(java.nio.file.StandardOpenOption.READ) ByteBuffer.allocate(java.nio.ByteBuffer.allocate) Files.createTempDirectory(java.nio.file.Files.createTempDirectory) EnumSet.allOf(java.util.EnumSet.allOf) AeronArchive(io.aeron.archive.client.AeronArchive) Path(java.nio.file.Path) WRITE(java.nio.file.StandardOpenOption.WRITE) MethodSource(org.junit.jupiter.params.provider.MethodSource) PrintStream(java.io.PrintStream) MutableBoolean(org.agrona.collections.MutableBoolean) LogBufferDescriptor.positionBitsToShift(io.aeron.logbuffer.LogBufferDescriptor.positionBitsToShift) ArchiveTool(io.aeron.archive.ArchiveTool) Collections.emptySet(java.util.Collections.emptySet) VERIFY_ALL_SEGMENT_FILES(io.aeron.archive.ArchiveTool.VerifyOption.VERIFY_ALL_SEGMENT_FILES) IOException(java.io.IOException) Mockito.times(org.mockito.Mockito.times) Checksum(io.aeron.archive.checksum.Checksum) Arguments(org.junit.jupiter.params.provider.Arguments) File(java.io.File) Mockito(org.mockito.Mockito) INVALID(io.aeron.archive.codecs.RecordingState.INVALID) VALID(io.aeron.archive.codecs.RecordingState.VALID) ParameterizedTest(org.junit.jupiter.params.ParameterizedTest) List(java.util.List) Collections.addAll(java.util.Collections.addAll) org.junit.jupiter.api(org.junit.jupiter.api) Executable(org.junit.jupiter.api.function.Executable) APPLY_CHECKSUM(io.aeron.archive.ArchiveTool.VerifyOption.APPLY_CHECKSUM) EnumSet.of(java.util.EnumSet.of) Assertions(org.junit.jupiter.api.Assertions) RecordingState(io.aeron.archive.codecs.RecordingState) EpochClock(org.agrona.concurrent.EpochClock) DataHeaderFlyweight(io.aeron.protocol.DataHeaderFlyweight) Checksums.crc32(io.aeron.archive.checksum.Checksums.crc32) Archive.segmentFileName(io.aeron.archive.Archive.segmentFileName) FileChannel(java.nio.channels.FileChannel) Files.deleteIfExists(java.nio.file.Files.deleteIfExists) LogBufferDescriptor.computeTermIdFromPosition(io.aeron.logbuffer.LogBufferDescriptor.computeTermIdFromPosition) Catalog(io.aeron.archive.Catalog) ParameterizedTest(org.junit.jupiter.params.ParameterizedTest)

Aggregations

Catalog (io.aeron.archive.Catalog)98 ParameterizedTest (org.junit.jupiter.params.ParameterizedTest)78 Test (org.junit.jupiter.api.Test)52 File (java.io.File)44 Checksum (io.aeron.archive.checksum.Checksum)40 ByteBuffer (java.nio.ByteBuffer)36 FileChannel (java.nio.channels.FileChannel)36 DataHeaderFlyweight (io.aeron.protocol.DataHeaderFlyweight)34 IOException (java.io.IOException)34 INVALID (io.aeron.archive.codecs.RecordingState.INVALID)30 VALID (io.aeron.archive.codecs.RecordingState.VALID)30 EpochClock (org.agrona.concurrent.EpochClock)30 Archive.segmentFileName (io.aeron.archive.Archive.segmentFileName)26 Checksums.crc32 (io.aeron.archive.checksum.Checksums.crc32)26 ByteBuffer.allocate (java.nio.ByteBuffer.allocate)26 IoUtil (org.agrona.IoUtil)26 Assertions (org.junit.jupiter.api.Assertions)26 Arguments (org.junit.jupiter.params.provider.Arguments)26 MethodSource (org.junit.jupiter.params.provider.MethodSource)26 Path (java.nio.file.Path)24