Examples with DateRangeFilter com.hedera.mirror.importer.config.MirrorDateRangePropertiesProcessor.DateRangeFilter used on opensource projects

Example 1 with DateRangeFilter

use of com.hedera.mirror.importer.config.MirrorDateRangePropertiesProcessor.DateRangeFilter in project hedera-mirror-node by hashgraph.

the class RecordFileParserTest method endDate.

@ParameterizedTest(name = "endDate with offset {0}ns")
@CsvSource({ "-1", "0", "1" })
void endDate(long offset) {
    // given
    RecordFile recordFile = (RecordFile) getStreamFile();
    RecordItem firstItem = recordFile.getItems().blockFirst();
    long end = recordFile.getConsensusStart() + offset;
    DateRangeFilter filter = new DateRangeFilter(Instant.EPOCH, Instant.ofEpochSecond(0, end));
    doReturn(filter).when(mirrorDateRangePropertiesProcessor).getDateRangeFilter(parserProperties.getStreamType());
    // when
    parser.parse(recordFile);
    // then
    verify(recordStreamFileListener).onStart();
    if (offset >= 0) {
        verify(recordItemListener).onItem(firstItem);
    }
    verify(recordStreamFileListener).onEnd(recordFile);
    assertPostParseStreamFile(recordFile, true);
}

Example 2 with DateRangeFilter

use of com.hedera.mirror.importer.config.MirrorDateRangePropertiesProcessor.DateRangeFilter in project hedera-mirror-node by hashgraph.

the class RecordFileParserTest method startDate.

@ParameterizedTest(name = "startDate with offset {0}ns")
@CsvSource({ "-1", "0", "1" })
void startDate(long offset) {
    // given
    RecordFile recordFile = (RecordFile) getStreamFile();
    RecordItem firstItem = recordFile.getItems().blockFirst();
    long start = recordFile.getConsensusStart() + offset;
    DateRangeFilter filter = new DateRangeFilter(Instant.ofEpochSecond(0, start), null);
    doReturn(filter).when(mirrorDateRangePropertiesProcessor).getDateRangeFilter(parserProperties.getStreamType());
    // when
    parser.parse(recordFile);
    // then
    verify(recordStreamFileListener).onStart();
    if (offset < 0) {
        verify(recordItemListener).onItem(firstItem);
    }
    verify(recordStreamFileListener).onEnd(recordFile);
    assertPostParseStreamFile(recordFile, true);
}

Example 3 with DateRangeFilter

use of com.hedera.mirror.importer.config.MirrorDateRangePropertiesProcessor.DateRangeFilter in project hedera-mirror-node by hashgraph.

the class ErrataMigration method missingTransactions.

// Adds the transactions and records that are missing due to the insufficient fee funding issue in services.
private void missingTransactions() throws IOException {
    Set<Long> consensusTimestamps = new HashSet<>();
    var resourceResolver = new PathMatchingResourcePatternResolver();
    Resource[] resources = resourceResolver.getResources("classpath*:errata/mainnet/missingtransactions/*.bin");
    recordStreamFileListener.onStart();
    var dateRangeFilter = new DateRangeFilter(mirrorProperties.getStartDate(), mirrorProperties.getEndDate());
    for (Resource resource : resources) {
        String name = resource.getFilename();
        log.info("Loading file: {}", name);
        try (var in = new ValidatedDataInputStream(resource.getInputStream(), name)) {
            byte[] recordBytes = in.readLengthAndBytes(1, MAX_TRANSACTION_LENGTH, false, "record");
            byte[] transactionBytes = in.readLengthAndBytes(1, MAX_TRANSACTION_LENGTH, false, "transaction");
            var transactionRecord = TransactionRecord.parseFrom(recordBytes);
            var transaction = Transaction.parseFrom(transactionBytes);
            var recordItem = new RecordItem(transaction, transactionRecord);
            long timestamp = recordItem.getConsensusTimestamp();
            if (transactionRepository.findById(timestamp).isEmpty() && dateRangeFilter.filter(timestamp)) {
                entityRecordItemListener.onItem(recordItem);
                consensusTimestamps.add(timestamp);
            }
        } catch (IOException e) {
            recordStreamFileListener.onError();
            throw new FileOperationException("Error parsing errata file " + name, e);
        }
    }
    if (consensusTimestamps.isEmpty()) {
        log.info("Previously inserted all missing transactions");
        return;
    }
    recordStreamFileListener.onEnd(null);
    var ids = new MapSqlParameterSource("ids", consensusTimestamps);
    jdbcOperations.update("update crypto_transfer set errata = 'INSERT' where consensus_timestamp in (:ids)", ids);
    jdbcOperations.update("update transaction set errata = 'INSERT' where consensus_timestamp in (:ids)", ids);
    Long min = consensusTimestamps.stream().min(Long::compareTo).orElse(null);
    Long max = consensusTimestamps.stream().max(Long::compareTo).orElse(null);
    log.info("Inserted {} missing transactions between {} and {}", consensusTimestamps.size(), min, max);
}

Example 4 with DateRangeFilter

use of com.hedera.mirror.importer.config.MirrorDateRangePropertiesProcessor.DateRangeFilter in project hedera-mirror-node by hashgraph.

the class MirrorDateRangePropertiesProcessorTest method startDateSetAndDatabaseEmpty.

@Test
void startDateSetAndDatabaseEmpty() {
    var startDate = STARTUP_TIME.plusSeconds(10L);
    mirrorProperties.setStartDate(startDate);
    DateRangeFilter expectedFilter = new DateRangeFilter(mirrorProperties.getStartDate(), null);
    Instant expectedDate = mirrorProperties.getStartDate();
    for (var downloaderProperties : downloaderPropertiesList) {
        StreamType streamType = downloaderProperties.getStreamType();
        assertThat(mirrorDateRangePropertiesProcessor.getLastStreamFile(streamType)).isEqualTo(streamFile(streamType, expectedDate));
        assertThat(mirrorDateRangePropertiesProcessor.getDateRangeFilter(streamType)).isEqualTo(expectedFilter);
    }
    assertThat(mirrorProperties.getVerifyHashAfter()).isEqualTo(startDate);
}

Example 5 with DateRangeFilter

use of com.hedera.mirror.importer.config.MirrorDateRangePropertiesProcessor.DateRangeFilter in project hedera-mirror-node by hashgraph.

the class MirrorDateRangePropertiesProcessorTest method startDateNotSetAndEndDateAfterLongMaxAndDatabaseNotEmpty.

@Test
void startDateNotSetAndEndDateAfterLongMaxAndDatabaseNotEmpty() {
    Instant past = STARTUP_TIME.minusSeconds(100);
    mirrorProperties.setEndDate(Utility.MAX_INSTANT_LONG.plusNanos(1));
    doReturn(streamFile(StreamType.BALANCE, past)).when(accountBalanceFileRepository).findLatest();
    doReturn(streamFile(StreamType.EVENT, past)).when(eventFileRepository).findLatest();
    doReturn(streamFile(StreamType.RECORD, past)).when(recordFileRepository).findLatest();
    for (var downloaderProperties : downloaderPropertiesList) {
        StreamType streamType = downloaderProperties.getStreamType();
        assertThat(mirrorDateRangePropertiesProcessor.getLastStreamFile(streamType)).matches(s -> matches(s, past));
        assertThat(mirrorDateRangePropertiesProcessor.getDateRangeFilter(streamType)).isEqualTo(new DateRangeFilter(past, Utility.MAX_INSTANT_LONG));
    }
    assertThat(mirrorProperties.getVerifyHashAfter()).isEqualTo(Instant.EPOCH);
}