Examples with Catalog io.aeron.archive.Catalog used on opensource projects

Example 56 with Catalog

use of io.aeron.archive.Catalog in project Aeron by real-logic.

the class ArchiveTool method dump.

/**
 * Dump the contents of an {@link Archive} so it can be inspected or debugged. Each recording will have its
 * contents dumped up to fragmentCountLimit before requesting to continue.
 *
 * @param out                               to which the contents will be printed.
 * @param archiveDir                        containing the {@link Archive}.
 * @param fragmentCountLimit                limit of data fragments to print from each recording before continue.
 * @param confirmActionOnFragmentCountLimit confirm continue to dump at limit.
 */
public static void dump(final PrintStream out, final File archiveDir, final long fragmentCountLimit, final ActionConfirmation<Long> confirmActionOnFragmentCountLimit) {
    try (Catalog catalog = openCatalogReadOnly(archiveDir, INSTANCE);
        ArchiveMarkFile markFile = openMarkFile(archiveDir, out::println)) {
        printMarkInformation(markFile, out);
        out.println("Catalog capacity in bytes: " + catalog.capacity());
        out.println();
        out.println("Dumping up to " + fragmentCountLimit + " fragments per recording");
        catalog.forEach((recordingDescriptorOffset, headerEncoder, headerDecoder, descriptorEncoder, descriptorDecoder) -> dump(out, archiveDir, catalog, fragmentCountLimit, confirmActionOnFragmentCountLimit, headerDecoder, descriptorDecoder));
    }
}

Example 57 with Catalog

use of io.aeron.archive.Catalog in project Aeron by real-logic.

the class ArchiveTool method compact.

static void compact(final PrintStream out, final File archiveDir, final EpochClock epochClock) {
    final File compactFile = new File(archiveDir, CATALOG_FILE_NAME + ".compact");
    try {
        final Path compactFilePath = compactFile.toPath();
        try (FileChannel channel = FileChannel.open(compactFilePath, READ, WRITE, CREATE_NEW);
            Catalog catalog = openCatalogReadOnly(archiveDir, epochClock)) {
            final MappedByteBuffer mappedByteBuffer = channel.map(READ_WRITE, 0, MAX_CATALOG_LENGTH);
            mappedByteBuffer.order(CatalogHeaderEncoder.BYTE_ORDER);
            try {
                final UnsafeBuffer unsafeBuffer = new UnsafeBuffer(mappedByteBuffer);
                new CatalogHeaderEncoder().wrap(unsafeBuffer, 0).version(catalog.version()).length(CatalogHeaderEncoder.BLOCK_LENGTH).nextRecordingId(catalog.nextRecordingId()).alignment(catalog.alignment());
                final MutableInteger offset = new MutableInteger(CatalogHeaderEncoder.BLOCK_LENGTH);
                final MutableInteger deletedRecords = new MutableInteger();
                final MutableInteger reclaimedBytes = new MutableInteger();
                catalog.forEach((recordingDescriptorOffset, headerEncoder, headerDecoder, descriptorEncoder, descriptorDecoder) -> {
                    final int frameLength = headerDecoder.encodedLength() + headerDecoder.length();
                    if (INVALID == headerDecoder.state()) {
                        deletedRecords.increment();
                        reclaimedBytes.addAndGet(frameLength);
                        final String[] segmentFiles = listSegmentFiles(archiveDir, descriptorDecoder.recordingId());
                        if (segmentFiles != null) {
                            for (final String segmentFile : segmentFiles) {
                                IoUtil.deleteIfExists(new File(archiveDir, segmentFile));
                            }
                        }
                    } else {
                        final int index = offset.getAndAdd(frameLength);
                        unsafeBuffer.putBytes(index, headerDecoder.buffer(), 0, frameLength);
                    }
                });
                out.println("Compaction result: deleted " + deletedRecords.get() + " records and reclaimed " + reclaimedBytes.get() + " bytes");
            } finally {
                BufferUtil.free(mappedByteBuffer);
            }
        }
        final Path catalogFilePath = compactFilePath.resolveSibling(CATALOG_FILE_NAME);
        Files.delete(catalogFilePath);
        Files.move(compactFilePath, catalogFilePath);
    } catch (final IOException ex) {
        ex.printStackTrace(out);
    } finally {
        IoUtil.deleteIfExists(compactFile);
    }
}

Example 58 with Catalog

use of io.aeron.archive.Catalog in project Aeron by real-logic.

the class ArchiveToolTests method deleteOrphanedSegmentsDeletesSegmentFilesForAllRecordings.

@Test
void deleteOrphanedSegmentsDeletesSegmentFilesForAllRecordings() throws IOException {
    final long rec1;
    final long rec2;
    try (Catalog catalog = new Catalog(archiveDir, epochClock, 1024, true, null, null)) {
        rec1 = catalog.addNewRecording(0, NULL_POSITION, NULL_TIMESTAMP, NULL_TIMESTAMP, 0, SEGMENT_LENGTH, TERM_LENGTH, MTU_LENGTH, 42, 5, "some ch", "some ch", "rec1");
        rec2 = catalog.addNewRecording(1_000_000, 1024 * 1024 * 1024, NULL_TIMESTAMP, NULL_TIMESTAMP, 0, SEGMENT_LENGTH, TERM_LENGTH, MTU_LENGTH, 1, 1, "ch2", "ch2", "rec2");
        catalog.invalidateRecording(rec2);
    }
    final File file11 = createFile(segmentFileName(rec1, -1));
    final File file12 = createFile(segmentFileName(rec1, 0));
    final File file13 = createFile(segmentFileName(rec1, Long.MAX_VALUE));
    final File file14 = createFile(rec1 + "-will-be-deleted.rec");
    final File file15 = createFile(rec1 + "-will-be-skipped.txt");
    final File file16 = createFile(rec1 + "-.rec");
    final File file17 = createFile(rec1 + "invalid_file_name.rec");
    final File file21 = createFile(segmentFileName(rec2, 0));
    final File file22 = createFile(segmentFileName(rec2, segmentFileBasePosition(1_000_000, 1_000_000, TERM_LENGTH, SEGMENT_LENGTH)));
    final File file23 = createFile(segmentFileName(rec2, segmentFileBasePosition(1_000_000, 5_000_000, TERM_LENGTH, SEGMENT_LENGTH)));
    final File file24 = createFile(segmentFileName(rec2, segmentFileBasePosition(1_000_000, 1024 * 1024 * 1024, TERM_LENGTH, SEGMENT_LENGTH)));
    final File file25 = createFile(segmentFileName(rec2, segmentFileBasePosition(1_000_000, Long.MAX_VALUE, TERM_LENGTH, SEGMENT_LENGTH)));
    deleteOrphanedSegments(out, archiveDir, epochClock);
    assertFileExists(file12, file13, file15, file17);
    assertFileDoesNotExist(file11, file14, file16);
    assertFileExists(file22, file23, file24);
    assertFileDoesNotExist(file21, file25);
}

Example 59 with Catalog

use of io.aeron.archive.Catalog in project Aeron by real-logic.

the class ArchiveToolTests method verifyAllOptionsTruncateOnPageStraddle.

@Test
void verifyAllOptionsTruncateOnPageStraddle() {
    final Checksum checksum = crc32();
    try (Catalog catalog = openCatalogReadWrite(archiveDir, epochClock, MIN_CAPACITY, checksum, null)) {
        catalog.forEach((recordingDescriptorOffset, headerEncoder, headerDecoder, descriptorEncoder, descriptorDecoder) -> catalog.updateChecksum(recordingDescriptorOffset));
    }
    assertFalse(verify(out, archiveDir, allOf(VerifyOption.class), checksum, epochClock, (file) -> true));
    try (Catalog catalog = openCatalogReadOnly(archiveDir, epochClock)) {
        assertRecording(catalog, invalidRecording0, INVALID, -119969720, NULL_POSITION, NULL_POSITION, 1, NULL_TIMESTAMP, 0, 1, "ch1", "src1");
        assertRecording(catalog, invalidRecording1, INVALID, 768794941, FRAME_ALIGNMENT - 7, NULL_POSITION, 2, NULL_TIMESTAMP, 0, 1, "ch1", "src1");
        assertRecording(catalog, invalidRecording2, INVALID, -1340428433, 1024, FRAME_ALIGNMENT * 2, 3, NULL_TIMESTAMP, 0, 1, "ch1", "src1");
        assertRecording(catalog, invalidRecording3, INVALID, 1464972620, 0, FRAME_ALIGNMENT * 5 + 11, 4, NULL_TIMESTAMP, 0, 1, "ch1", "src1");
        assertRecording(catalog, invalidRecording4, INVALID, 21473288, SEGMENT_LENGTH, NULL_POSITION, 5, NULL_TIMESTAMP, 0, 1, "ch1", "src1");
        assertRecording(catalog, invalidRecording5, INVALID, -2119992405, 0, SEGMENT_LENGTH, 6, NULL_TIMESTAMP, 0, 1, "ch1", "src1");
        assertRecording(catalog, invalidRecording6, INVALID, 2054096463, 0, NULL_POSITION, 7, NULL_TIMESTAMP, 0, 1, "ch1", "src1");
        assertRecording(catalog, invalidRecording7, INVALID, -1050175867, 0, NULL_POSITION, 8, NULL_TIMESTAMP, 0, 1, "ch1", "src1");
        assertRecording(catalog, invalidRecording8, INVALID, -504693275, 0, NULL_POSITION, 9, NULL_TIMESTAMP, 0, 1, "ch1", "src1");
        assertRecording(catalog, invalidRecording9, INVALID, -2036430506, 0, NULL_POSITION, 10, NULL_TIMESTAMP, 0, 1, "ch1", "src1");
        assertRecording(catalog, invalidRecording10, INVALID, -414736820, 128, NULL_POSITION, 11, NULL_TIMESTAMP, 0, 1, "ch1", "src1");
        assertRecording(catalog, invalidRecording11, INVALID, 1983095657, 0, NULL_POSITION, 12, NULL_TIMESTAMP, 5, 1, "ch1", "src1");
        assertRecording(catalog, invalidRecording12, INVALID, -1308504240, 0, NULL_POSITION, 13, NULL_TIMESTAMP, 9, 6, "ch1", "src1");
        assertRecording(catalog, invalidRecording13, INVALID, -273182324, 0, NULL_POSITION, 14, NULL_TIMESTAMP, 0, 13, "ch1", "src1");
        assertRecording(catalog, invalidRecording14, INVALID, 213018412, 128, NULL_POSITION, -14, 41, -14, 0, "ch1", "src1");
        assertRecording(catalog, validRecording0, VALID, 356725588, 0, TERM_LENGTH + 64, 15, 100, 0, 2, "ch2", "src2");
        assertRecording(catalog, validRecording1, VALID, -1571032591, 1024, 1024, 16, 200, 0, 2, "ch2", "src2");
        assertRecording(catalog, validRecording2, VALID, 114203747, TERM_LENGTH * 3 + 96, TERM_LENGTH * 3 + 96, 17, 300, 0, 2, "ch2", "src2");
        assertRecording(catalog, validRecording3, INVALID, 963969455, 7 * TERM_LENGTH + 96, 7 * TERM_LENGTH + 128, 18, NULL_TIMESTAMP, 7, 13, "ch2", "src2");
        assertRecording(catalog, validRecording4, INVALID, 162247708, 21 * TERM_LENGTH + (TERM_LENGTH - 64), 22 * TERM_LENGTH + 992, 19, 1, -25, 7, "ch2", "src2");
        assertRecording(catalog, validRecording51, VALID, -940881948, 0, 64 + PAGE_SIZE, 20, 777, 0, 20, "ch2", "src2");
        assertRecording(catalog, validRecording52, INVALID, 1046083782, 0, NULL_POSITION, 21, NULL_TIMESTAMP, 0, 52, "ch2", "src2");
        assertRecording(catalog, validRecording53, INVALID, 428178649, 0, NULL_POSITION, 22, NULL_TIMESTAMP, 0, 53, "ch2", "src2");
        assertRecording(catalog, validRecording6, VALID, -175549265, 352, 960, 23, 400, 0, 6, "ch2", "src2");
    }
    Mockito.verify(out, times(24)).println(any(String.class));
}

Example 60 with Catalog

use of io.aeron.archive.Catalog in project Aeron by real-logic.

the class ArchiveToolTests method checksumLastSegmentFile.

@Test
void checksumLastSegmentFile() {
    checksum(out, archiveDir, false, crc32(), epochClock);
    assertFalse(verify(out, archiveDir, allOf(VerifyOption.class), crc32(), epochClock, (file) -> false));
    try (Catalog catalog = openCatalogReadOnly(archiveDir, epochClock)) {
        assertRecording(catalog, validRecording0, VALID, 356725588, 0, TERM_LENGTH + 64, 15, 100, 0, 2, "ch2", "src2");
        assertRecording(catalog, validRecording1, VALID, -1571032591, 1024, 1024, 16, 200, 0, 2, "ch2", "src2");
        assertRecording(catalog, validRecording2, VALID, 114203747, TERM_LENGTH * 3 + 96, TERM_LENGTH * 3 + 96, 17, 300, 0, 2, "ch2", "src2");
        assertRecording(catalog, validRecording3, INVALID, 963969455, 7 * TERM_LENGTH + 96, 7 * TERM_LENGTH + 128, 18, NULL_TIMESTAMP, 7, 13, "ch2", "src2");
        assertRecording(catalog, validRecording4, INVALID, 162247708, 21 * TERM_LENGTH + (TERM_LENGTH - 64), 22 * TERM_LENGTH + 992, 19, 1, -25, 7, "ch2", "src2");
        assertRecording(catalog, validRecording51, VALID, -940881948, 0, 64 + PAGE_SIZE, 20, 777, 0, 20, "ch2", "src2");
        assertRecording(catalog, validRecording6, VALID, -175549265, 352, 960, 23, 600, 0, 6, "ch2", "src2");
    }
}