Examples with DateRangeFilter com.hedera.mirror.importer.config.MirrorDateRangePropertiesProcessor.DateRangeFilter used on opensource projects

Example 11 with DateRangeFilter

use of com.hedera.mirror.importer.config.MirrorDateRangePropertiesProcessor.DateRangeFilter in project hedera-mirror-node by hashgraph.

the class MirrorDateRangePropertiesProcessorTest method filter.

@ParameterizedTest(name = "filter [{0}, {1}], timestamp {2}, pass: {3}")
@CsvSource(value = { "1, 1, 1, true", "1, 10, 1, true", "1, 10, 10, true", "1, 10, 6, true", "1, 10, 0, false", "1, 10, 11, false", "1, 10, -1, false" })
void filter(long start, long end, long timestamp, boolean expected) {
    DateRangeFilter filter = new DateRangeFilter(Instant.ofEpochSecond(0, start), Instant.ofEpochSecond(0, end));
    assertThat(filter.filter(timestamp)).isEqualTo(expected);
}

Example 12 with DateRangeFilter

use of com.hedera.mirror.importer.config.MirrorDateRangePropertiesProcessor.DateRangeFilter in project hedera-mirror-node by hashgraph.

the class MirrorDateRangePropertiesProcessorTest method notSetAndDatabaseEmpty.

@Test
void notSetAndDatabaseEmpty() {
    Instant expectedDate = STARTUP_TIME;
    DateRangeFilter expectedFilter = new DateRangeFilter(expectedDate, null);
    for (var downloaderProperties : downloaderPropertiesList) {
        StreamType streamType = downloaderProperties.getStreamType();
        assertThat(mirrorDateRangePropertiesProcessor.getLastStreamFile(streamType)).isEqualTo(streamFile(streamType, expectedDate));
        assertThat(mirrorDateRangePropertiesProcessor.getDateRangeFilter(streamType)).isEqualTo(expectedFilter);
    }
    assertThat(mirrorProperties.getVerifyHashAfter()).isEqualTo(expectedDate);
}

Example 13 with DateRangeFilter

use of com.hedera.mirror.importer.config.MirrorDateRangePropertiesProcessor.DateRangeFilter in project hedera-mirror-node by hashgraph.

the class AccountBalanceFileParser method doParse.

@Override
protected void doParse(AccountBalanceFile accountBalanceFile) {
    log.info("Starting processing account balances file {}", accountBalanceFile.getName());
    DateRangeFilter filter = mirrorDateRangePropertiesProcessor.getDateRangeFilter(StreamType.BALANCE);
    int batchSize = ((BalanceParserProperties) parserProperties).getBatchSize();
    long count = 0L;
    if (filter.filter(accountBalanceFile.getConsensusTimestamp())) {
        List<AccountBalance> accountBalances = new ArrayList<>(batchSize);
        Map<TokenBalance.Id, TokenBalance> tokenBalances = new HashMap<>(batchSize);
        count = accountBalanceFile.getItems().doOnNext(accountBalance -> {
            accountBalances.add(accountBalance);
            for (var tokenBalance : accountBalance.getTokenBalances()) {
                if (tokenBalances.putIfAbsent(tokenBalance.getId(), tokenBalance) != null) {
                    log.warn("Skipping duplicate token balance: {}", tokenBalance);
                }
            }
            if (accountBalances.size() >= batchSize) {
                batchPersister.persist(accountBalances);
                accountBalances.clear();
            }
            if (tokenBalances.size() >= batchSize) {
                batchPersister.persist(tokenBalances.values());
                tokenBalances.clear();
            }
        }).count().block();
        batchPersister.persist(accountBalances);
        batchPersister.persist(tokenBalances.values());
    }
    Instant loadEnd = Instant.now();
    accountBalanceFile.setCount(count);
    accountBalanceFile.setLoadEnd(loadEnd.getEpochSecond());
    streamFileListener.onEnd(accountBalanceFile);
    streamFileRepository.save(accountBalanceFile);
}

Example 14 with DateRangeFilter

use of com.hedera.mirror.importer.config.MirrorDateRangePropertiesProcessor.DateRangeFilter in project hedera-mirror-node by hashgraph.

the class RecordFileParser method doParse.

@Override
protected void doParse(RecordFile recordFile) {
    DateRangeFilter dateRangeFilter = mirrorDateRangePropertiesProcessor.getDateRangeFilter(parserProperties.getStreamType());
    try {
        Flux<RecordItem> recordItems = recordFile.getItems();
        if (log.getLevel().isInRange(Level.DEBUG, Level.TRACE)) {
            recordItems = recordItems.doOnNext(this::logItem);
        }
        recordStreamFileListener.onStart();
        long count = recordItems.doOnNext(recordFile::processItem).filter(r -> dateRangeFilter.filter(r.getConsensusTimestamp())).doOnNext(recordItemListener::onItem).doOnNext(this::recordMetrics).count().block();
        recordFile.finishLoad(count);
        recordStreamFileListener.onEnd(recordFile);
    } catch (Exception ex) {
        recordStreamFileListener.onError();
        throw ex;
    }
}