Examples with FileData com.hedera.mirror.common.domain.file.FileData used on opensource projects

Example 6 with FileData

use of com.hedera.mirror.common.domain.file.FileData in project hedera-mirror-node by hashgraph.

the class AddressBookServiceImpl method migrate.

/**
 * Migrates address book data by searching file_data table for applicable 101 and 102 files. These files are
 * converted to AddressBook objects and used to populate address_book and address_book_entry tables. Migrate flow
 * will populate initial addressBook where applicable and consider all file_data present
 *
 * @return Latest AddressBook from historical files
 */
@Override
public synchronized AddressBook migrate() {
    long consensusTimestamp = DomainUtils.convertToNanosMax(Instant.now());
    var currentAddressBook = addressBookRepository.findLatestAddressBook(consensusTimestamp, ADDRESS_BOOK_102_ENTITY_ID.getId()).orElse(null);
    if (currentAddressBook != null) {
        // verify no file_data 102 entries exists after current addressBook
        List<FileData> fileDataList = fileDataRepository.findAddressBooksBetween(currentAddressBook.getStartConsensusTimestamp(), Long.MAX_VALUE, 1);
        if (CollectionUtils.isEmpty(fileDataList)) {
            log.trace("All valid address books exist in db, skipping migration");
            return currentAddressBook;
        }
        log.warn("Valid address book file data entries exist in db after current address book");
    }
    log.info("Empty or incomplete list of address books found in db, proceeding with migration");
    return transactionTemplate.execute(status -> {
        log.info("Searching for address book on file system");
        var initialAddressBook = currentAddressBook == null ? parse(getInitialAddressBookFileData()) : currentAddressBook;
        // Parse all applicable addressBook file_data entries are processed
        AddressBook latestAddressBook = parseHistoricAddressBooks(initialAddressBook.getStartConsensusTimestamp() - 1, consensusTimestamp);
        // set latestAddressBook as newest addressBook from file_data entries or initial addressBook from filesystem
        latestAddressBook = latestAddressBook == null ? initialAddressBook : latestAddressBook;
        log.info("Migration complete. Current address book to db: {}", latestAddressBook);
        return latestAddressBook;
    });
}

Example 7 with FileData

use of com.hedera.mirror.common.domain.file.FileData in project hedera-mirror-node by hashgraph.

the class AddressBookServiceImpl method parseHistoricAddressBooks.

/**
 * Batch parse all 101 and 102 addressBook fileData objects and update the address_book table. Uses page size and
 * timestamp counters to batch query file data entries within timestamp range
 */
private AddressBook parseHistoricAddressBooks(long startTimestamp, long endTimestamp) {
    var fileDataEntries = 0;
    long currentConsensusTimestamp = startTimestamp;
    AddressBook lastAddressBook = null;
    // retrieve pages of fileData entries for historic address books within range
    var pageSize = 1000;
    List<FileData> fileDataList = fileDataRepository.findAddressBooksBetween(currentConsensusTimestamp, endTimestamp, pageSize);
    while (!CollectionUtils.isEmpty(fileDataList)) {
        log.info("Retrieved {} file_data rows for address book processing", fileDataList.size());
        for (FileData fileData : fileDataList) {
            if (fileData.getFileData() != null && fileData.getFileData().length > 0) {
                // convert and ingest address book fileData contents
                lastAddressBook = parse(fileData);
                fileDataEntries++;
            }
            // update timestamp counter to ensure next query doesn't reconsider files in this time range
            currentConsensusTimestamp = fileData.getConsensusTimestamp();
        }
        fileDataList = fileDataRepository.findAddressBooksBetween(currentConsensusTimestamp, endTimestamp, pageSize);
    }
    log.info("Processed {} historic address books", fileDataEntries);
    return lastAddressBook;
}

Example 8 with FileData

use of com.hedera.mirror.common.domain.file.FileData in project hedera-mirror-node by hashgraph.

the class AddressBookServiceImpl method combinePreviousFileDataContents.

/**
 * Concatenates byte arrays of first fileCreate/fileUpdate transaction and intermediate fileAppend entries that make
 * up the potential addressBook
 *
 * @param fileData file data entry containing address book bytes
 * @return
 */
private byte[] combinePreviousFileDataContents(FileData fileData) {
    FileData firstPartialAddressBook = fileDataRepository.findLatestMatchingFile(fileData.getConsensusTimestamp(), fileData.getEntityId().getId(), List.of(TransactionType.FILECREATE.getProtoId(), TransactionType.FILEUPDATE.getProtoId())).orElseThrow(() -> new IllegalStateException("Missing FileData entry. FileAppend expects a corresponding  FileCreate/FileUpdate entry"));
    List<FileData> appendFileDataEntries = fileDataRepository.findFilesInRange(getAddressBookStartConsensusTimestamp(firstPartialAddressBook), fileData.getConsensusTimestamp() - 1, firstPartialAddressBook.getEntityId().getId(), TransactionType.FILEAPPEND.getProtoId());
    try (var bos = new ByteArrayOutputStream(firstPartialAddressBook.getFileData().length)) {
        bos.write(firstPartialAddressBook.getFileData());
        for (var i = 0; i < appendFileDataEntries.size(); i++) {
            bos.write(appendFileDataEntries.get(i).getFileData());
        }
        bos.write(fileData.getFileData());
        return bos.toByteArray();
    } catch (IOException ex) {
        throw new InvalidDatasetException("Error concatenating partial address book fileData entries", ex);
    }
}

Example 9 with FileData

use of com.hedera.mirror.common.domain.file.FileData in project hedera-mirror-node by hashgraph.

the class AddressBookServiceImpl method getInitialAddressBookFileData.

/**
 * Retrieve the initial address book file for the network from either the file system or class path
 *
 * @return Address book fileData object
 */
private FileData getInitialAddressBookFileData() {
    byte[] addressBookBytes;
    // retrieve bootstrap address book from filesystem or classpath
    try {
        Path initialAddressBook = mirrorProperties.getInitialAddressBook();
        if (initialAddressBook != null) {
            log.info("Loading bootstrap address book from {}", initialAddressBook.toString());
            addressBookBytes = Files.readAllBytes(initialAddressBook);
        } else {
            var hederaNetwork = mirrorProperties.getNetwork();
            var resourcePath = String.format("/addressbook/%s", hederaNetwork.name().toLowerCase());
            log.info("Loading bootstrap address book from {}", resourcePath);
            Resource resource = new ClassPathResource(resourcePath, getClass());
            addressBookBytes = IOUtils.toByteArray(resource.getInputStream());
        }
        log.info("Loaded bootstrap address book of {} B", addressBookBytes.length);
    } catch (Exception e) {
        throw new IllegalStateException("Unable to load bootstrap address book", e);
    }
    return new FileData(0L, addressBookBytes, AddressBookServiceImpl.ADDRESS_BOOK_102_ENTITY_ID, TransactionType.FILECREATE.getProtoId());
}

Example 10 with FileData

use of com.hedera.mirror.common.domain.file.FileData in project hedera-mirror-node by hashgraph.

the class AddressBookServiceImplTest method append.

private void append(byte[] contents, long consensusTimeStamp, boolean is102) {
    FileData fileData = createFileData(contents, consensusTimeStamp, is102, TransactionType.FILEAPPEND);
    fileDataRepository.save(fileData);
    addressBookService.update(fileData);
}