Examples with AccountBalanceFile com.hedera.mirror.common.domain.balance.AccountBalanceFile used on opensource projects

Example 16 with AccountBalanceFile

use of com.hedera.mirror.common.domain.balance.AccountBalanceFile in project hedera-mirror-node by hashgraph.

the class ProtoBalanceFileReaderTest method getExpectedAccountBalanceFile.

private AccountBalanceFile getExpectedAccountBalanceFile(StreamFileData streamFileData) {
    Instant instant = Instant.parse(TIMESTAMP.replace("_", ":"));
    long consensusTimestamp = DomainUtils.convertToNanosMax(instant);
    long accountNum = 2000;
    long hbarBalance = 3000;
    long tokenNum = 5000;
    long tokenBalance = 6000;
    List<AccountBalance> accountBalances = IntStream.range(0, 10).mapToObj(i -> {
        EntityId accountId = EntityId.of(0, 0, accountNum + i, EntityType.ACCOUNT);
        List<TokenBalance> tokenBalances = IntStream.range(0, 5).mapToObj(j -> {
            EntityId tokenId = EntityId.of(0, 0, tokenNum + i * 5 + j, EntityType.TOKEN);
            return new TokenBalance(tokenBalance + i * 5 + j, new TokenBalance.Id(consensusTimestamp, accountId, tokenId));
        }).collect(Collectors.toList());
        return new AccountBalance(hbarBalance + i, tokenBalances, new AccountBalance.Id(consensusTimestamp, accountId));
    }).collect(Collectors.toList());
    return AccountBalanceFile.builder().bytes(streamFileData.getBytes()).consensusTimestamp(consensusTimestamp).fileHash("67c2fd054621366dd5a37b6ee36a51bc590361379d539fdac2265af08cb8097729218c7d9ff1f1e354c85b820c5b8cf8").items(Flux.fromIterable(accountBalances)).name(streamFileData.getFilename()).build();
}

Example 17 with AccountBalanceFile

use of com.hedera.mirror.common.domain.balance.AccountBalanceFile in project hedera-mirror-node by hashgraph.

the class ProtoBalanceFileReaderTest method unknownFields.

@Test
void unknownFields() {
    UnknownFieldSet.Field field = UnknownFieldSet.Field.newBuilder().addFixed32(11).build();
    AllAccountBalances allAccountBalances = AllAccountBalances.newBuilder().setConsensusTimestamp(Timestamp.newBuilder().setSeconds(1L).build()).mergeUnknownFields(UnknownFieldSet.newBuilder().addField(23, field).build()).addAllAccounts(SingleAccountBalances.newBuilder().build()).build();
    byte[] bytes = allAccountBalances.toByteArray();
    StreamFileData streamFileData = StreamFileData.from(TIMESTAMP + "_Balances.pb", bytes);
    AccountBalanceFile accountBalanceFile = protoBalanceFileReader.read(streamFileData);
    assertThat(accountBalanceFile).isNotNull();
    assertThat(accountBalanceFile.getItems().count().block()).isEqualTo(1L);
}

Example 18 with AccountBalanceFile

use of com.hedera.mirror.common.domain.balance.AccountBalanceFile in project hedera-mirror-node by hashgraph.

the class ProtoBalanceFileReaderTest method readGzippedProtoBalanceFile.

@Test
void readGzippedProtoBalanceFile() {
    AccountBalanceFile actual = protoBalanceFileReader.read(streamFileData);
    assertThat(actual).usingRecursiveComparison().ignoringFields("loadStart", "nodeAccountId", "items").isEqualTo(expected);
    assertThat(expected.getItems().collectList().block()).isEqualTo(actual.getItems().collectList().block());
    assertThat(actual.getLoadStart()).isNotNull().isPositive();
}

Example 19 with AccountBalanceFile

use of com.hedera.mirror.common.domain.balance.AccountBalanceFile in project hedera-mirror-node by hashgraph.

the class CsvBalanceFileReader method read.

@Override
public AccountBalanceFile read(StreamFileData streamFileData) {
    MessageDigest messageDigest = DigestUtils.getSha384Digest();
    int bufferSize = balanceParserProperties.getFileBufferSize();
    try (InputStream inputStream = new DigestInputStream(streamFileData.getInputStream(), messageDigest);
        BufferedReader reader = new BufferedReader(new InputStreamReader(inputStream, CHARSET), bufferSize)) {
        long consensusTimestamp = parseConsensusTimestamp(reader);
        AtomicLong count = new AtomicLong(0L);
        List<AccountBalance> items = new ArrayList<>();
        AccountBalanceFile accountBalanceFile = new AccountBalanceFile();
        accountBalanceFile.setBytes(streamFileData.getBytes());
        accountBalanceFile.setConsensusTimestamp(consensusTimestamp);
        accountBalanceFile.setLoadStart(Instant.now().getEpochSecond());
        accountBalanceFile.setName(streamFileData.getFilename());
        reader.lines().map(line -> {
            try {
                AccountBalance accountBalance = parser.parse(line, consensusTimestamp);
                count.incrementAndGet();
                return accountBalance;
            } catch (InvalidDatasetException ex) {
                log.error(ex);
                return null;
            }
        }).filter(Objects::nonNull).forEachOrdered(items::add);
        accountBalanceFile.setCount(count.get());
        accountBalanceFile.setFileHash(DomainUtils.bytesToHex(messageDigest.digest()));
        accountBalanceFile.setItems(Flux.fromIterable(items));
        return accountBalanceFile;
    } catch (IOException ex) {
        throw new InvalidDatasetException("Error reading account balance file", ex);
    }
}

Example 20 with AccountBalanceFile

use of com.hedera.mirror.common.domain.balance.AccountBalanceFile in project hedera-mirror-node by hashgraph.

the class AccountBalanceFileParser method doParse.

@Override
protected void doParse(AccountBalanceFile accountBalanceFile) {
    log.info("Starting processing account balances file {}", accountBalanceFile.getName());
    DateRangeFilter filter = mirrorDateRangePropertiesProcessor.getDateRangeFilter(StreamType.BALANCE);
    int batchSize = ((BalanceParserProperties) parserProperties).getBatchSize();
    long count = 0L;
    if (filter.filter(accountBalanceFile.getConsensusTimestamp())) {
        List<AccountBalance> accountBalances = new ArrayList<>(batchSize);
        Map<TokenBalance.Id, TokenBalance> tokenBalances = new HashMap<>(batchSize);
        count = accountBalanceFile.getItems().doOnNext(accountBalance -> {
            accountBalances.add(accountBalance);
            for (var tokenBalance : accountBalance.getTokenBalances()) {
                if (tokenBalances.putIfAbsent(tokenBalance.getId(), tokenBalance) != null) {
                    log.warn("Skipping duplicate token balance: {}", tokenBalance);
                }
            }
            if (accountBalances.size() >= batchSize) {
                batchPersister.persist(accountBalances);
                accountBalances.clear();
            }
            if (tokenBalances.size() >= batchSize) {
                batchPersister.persist(tokenBalances.values());
                tokenBalances.clear();
            }
        }).count().block();
        batchPersister.persist(accountBalances);
        batchPersister.persist(tokenBalances.values());
    }
    Instant loadEnd = Instant.now();
    accountBalanceFile.setCount(count);
    accountBalanceFile.setLoadEnd(loadEnd.getEpochSecond());
    streamFileListener.onEnd(accountBalanceFile);
    streamFileRepository.save(accountBalanceFile);
}