Search in sources :

Example 71 with EntityId

use of com.hedera.mirror.common.domain.entity.EntityId in project hedera-mirror-node by hashgraph.

the class AddressBookServiceImpl method retrieveNodeAddressesFromAddressBook.

/**
 * Extracts a collection of AddressBookEntry domain objects from NodeAddressBook proto. Sets provided
 * consensusTimestamp as the consensusTimestamp of each address book entry to ensure mapping to a single
 * AddressBook
 *
 * @param nodeAddressBook    node address book proto
 * @param consensusTimestamp transaction consensusTimestamp
 * @return
 */
private static Collection<AddressBookEntry> retrieveNodeAddressesFromAddressBook(NodeAddressBook nodeAddressBook, long consensusTimestamp) throws UnknownHostException {
    // node id to entry
    Map<Long, AddressBookEntry> addressBookEntries = new LinkedHashMap<>();
    for (NodeAddress nodeAddressProto : nodeAddressBook.getNodeAddressList()) {
        Pair<Long, EntityId> nodeIds = getNodeIds(nodeAddressProto);
        AddressBookEntry addressBookEntry = addressBookEntries.computeIfAbsent(nodeIds.getLeft(), k -> getAddressBookEntry(nodeAddressProto, consensusTimestamp, nodeIds));
        Set<AddressBookServiceEndpoint> updatedList = new HashSet<>(addressBookEntry.getServiceEndpoints());
        updatedList.addAll(getAddressBookServiceEndpoints(nodeAddressProto, consensusTimestamp));
        addressBookEntry.setServiceEndpoints(updatedList);
    }
    return addressBookEntries.values();
}
Also used : EntityId(com.hedera.mirror.common.domain.entity.EntityId) AddressBookServiceEndpoint(com.hedera.mirror.common.domain.addressbook.AddressBookServiceEndpoint) AddressBookEntry(com.hedera.mirror.common.domain.addressbook.AddressBookEntry) NodeAddress(com.hederahashgraph.api.proto.java.NodeAddress) LinkedHashMap(java.util.LinkedHashMap) HashSet(java.util.HashSet)

Example 72 with EntityId

use of com.hedera.mirror.common.domain.entity.EntityId in project hedera-mirror-node by hashgraph.

the class AddressBookRepositoryTest method findLatestTimestamp.

@Test
void findLatestTimestamp() {
    EntityId fileId = EntityId.of(101L, EntityType.FILE);
    assertThat(addressBookRepository.findLatestTimestamp(fileId.getId())).isEmpty();
    domainBuilder.addressBook().customize(a -> a.fileId(EntityId.of(999L, EntityType.FILE))).persist();
    assertThat(addressBookRepository.findLatestTimestamp(fileId.getId())).isEmpty();
    AddressBook addressBook2 = domainBuilder.addressBook().customize(a -> a.fileId(fileId)).persist();
    assertThat(addressBookRepository.findLatestTimestamp(fileId.getId())).get().isEqualTo(addressBook2.getStartConsensusTimestamp());
    AddressBook addressBook3 = domainBuilder.addressBook().customize(a -> a.fileId(fileId)).persist();
    assertThat(addressBookRepository.findLatestTimestamp(fileId.getId())).get().isEqualTo(addressBook3.getStartConsensusTimestamp());
}
Also used : EntityId(com.hedera.mirror.common.domain.entity.EntityId) Test(org.junit.jupiter.api.Test) EntityId(com.hedera.mirror.common.domain.entity.EntityId) GrpcIntegrationTest(com.hedera.mirror.grpc.GrpcIntegrationTest) AddressBook(com.hedera.mirror.common.domain.addressbook.AddressBook) Assertions.assertThat(org.assertj.core.api.Assertions.assertThat) DomainBuilder(com.hedera.mirror.common.domain.DomainBuilder) Resource(javax.annotation.Resource) EntityType(com.hedera.mirror.common.domain.entity.EntityType) Transactional(org.springframework.transaction.annotation.Transactional) AddressBook(com.hedera.mirror.common.domain.addressbook.AddressBook) Test(org.junit.jupiter.api.Test) GrpcIntegrationTest(com.hedera.mirror.grpc.GrpcIntegrationTest)

Example 73 with EntityId

use of com.hedera.mirror.common.domain.entity.EntityId in project hedera-mirror-node by hashgraph.

the class Downloader method downloadAndParseSigFiles.

/**
 * Download and parse all signature files with a timestamp later than the last valid file. Put signature files into
 * a multi-map sorted and grouped by the timestamp.
 *
 * @param addressBook the current address book
 * @return a multi-map of signature file objects from different nodes, grouped by filename
 */
private Multimap<String, FileStreamSignature> downloadAndParseSigFiles(AddressBook addressBook) throws InterruptedException {
    String startAfterFilename = getStartAfterFilename();
    Multimap<String, FileStreamSignature> sigFilesMap = Multimaps.synchronizedSortedSetMultimap(TreeMultimap.create());
    Set<EntityId> nodeAccountIds = addressBook.getNodeSet();
    List<Callable<Object>> tasks = new ArrayList<>(nodeAccountIds.size());
    AtomicInteger totalDownloads = new AtomicInteger();
    log.info("Downloading signature files created after file: {}", startAfterFilename);
    /*
         * For each node, create a thread that will make S3 ListObject requests as many times as necessary to
         * start maxDownloads download operations.
         */
    for (EntityId nodeAccountId : nodeAccountIds) {
        tasks.add(Executors.callable(() -> {
            String nodeAccountIdStr = nodeAccountId.entityIdToString();
            Stopwatch stopwatch = Stopwatch.createStarted();
            try {
                List<S3Object> s3Objects = listFiles(startAfterFilename, nodeAccountIdStr);
                List<PendingDownload> pendingDownloads = downloadSignatureFiles(nodeAccountIdStr, s3Objects);
                AtomicInteger count = new AtomicInteger();
                pendingDownloads.forEach(pendingDownload -> {
                    try {
                        parseSignatureFile(pendingDownload, nodeAccountId).ifPresent(fileStreamSignature -> {
                            sigFilesMap.put(fileStreamSignature.getFilename(), fileStreamSignature);
                            count.incrementAndGet();
                            totalDownloads.incrementAndGet();
                        });
                    } catch (InterruptedException ex) {
                        log.warn("Failed downloading {} in {}", pendingDownload.getS3key(), pendingDownload.getStopwatch(), ex);
                        Thread.currentThread().interrupt();
                    } catch (Exception ex) {
                        log.warn("Failed to parse signature file {}: {}", pendingDownload.getS3key(), ex);
                    }
                });
                if (count.get() > 0) {
                    log.info("Downloaded {} signatures for node {} in {}", count.get(), nodeAccountIdStr, stopwatch);
                }
            } catch (InterruptedException e) {
                log.error("Error downloading signature files for node {} after {}", nodeAccountIdStr, stopwatch, e);
                Thread.currentThread().interrupt();
            } catch (Exception e) {
                log.error("Error downloading signature files for node {} after {}", nodeAccountIdStr, stopwatch, e);
            }
        }));
    }
    // Wait for all tasks to complete.
    // invokeAll() does return Futures, but it waits for all to complete (so they're returned in a completed state).
    Stopwatch stopwatch = Stopwatch.createStarted();
    signatureDownloadThreadPool.invokeAll(tasks);
    if (totalDownloads.get() > 0) {
        var rate = (int) (1000000.0 * totalDownloads.get() / stopwatch.elapsed(TimeUnit.MICROSECONDS));
        log.info("Downloaded {} signatures in {} ({}/s)", totalDownloads, stopwatch, rate);
    }
    return sigFilesMap;
}
Also used : EntityId(com.hedera.mirror.common.domain.entity.EntityId) AddressBook(com.hedera.mirror.common.domain.addressbook.AddressBook) FileStreamSignature(com.hedera.mirror.importer.domain.FileStreamSignature) TreeMultimap(com.google.common.collect.TreeMultimap) AtomicInteger(java.util.concurrent.atomic.AtomicInteger) Duration(java.time.Duration) Map(java.util.Map) GetObjectRequest(software.amazon.awssdk.services.s3.model.GetObjectRequest) AsyncResponseTransformer(software.amazon.awssdk.core.async.AsyncResponseTransformer) Path(java.nio.file.Path) Utility(com.hedera.mirror.importer.util.Utility) ListObjectsRequest(software.amazon.awssdk.services.s3.model.ListObjectsRequest) S3AsyncClient(software.amazon.awssdk.services.s3.S3AsyncClient) SIGNATURE(com.hedera.mirror.importer.domain.StreamFilename.FileType.SIGNATURE) Collection(java.util.Collection) StreamType(com.hedera.mirror.common.domain.StreamType) Set(java.util.Set) HashMismatchException(com.hedera.mirror.importer.exception.HashMismatchException) Instant(java.time.Instant) Collectors(java.util.stream.Collectors) Executors(java.util.concurrent.Executors) Objects(java.util.Objects) InvalidStreamFileException(com.hedera.mirror.importer.exception.InvalidStreamFileException) List(java.util.List) Logger(org.apache.logging.log4j.Logger) Optional(java.util.Optional) SHA384(com.hedera.mirror.common.domain.DigestAlgorithm.SHA384) ShutdownHelper(com.hedera.mirror.importer.util.ShutdownHelper) StreamFileData(com.hedera.mirror.importer.domain.StreamFileData) Stopwatch(com.google.common.base.Stopwatch) S3Object(software.amazon.awssdk.services.s3.model.S3Object) Collectors.groupingBy(java.util.stream.Collectors.groupingBy) Callable(java.util.concurrent.Callable) Multimap(com.google.common.collect.Multimap) AtomicReference(java.util.concurrent.atomic.AtomicReference) Multimaps(com.google.common.collect.Multimaps) ArrayList(java.util.ArrayList) RequestPayer(software.amazon.awssdk.services.s3.model.RequestPayer) AddressBookService(com.hedera.mirror.importer.addressbook.AddressBookService) Timer(io.micrometer.core.instrument.Timer) StreamFilename(com.hedera.mirror.importer.domain.StreamFilename) ExecutorService(java.util.concurrent.ExecutorService) MirrorDateRangePropertiesProcessor(com.hedera.mirror.importer.config.MirrorDateRangePropertiesProcessor) StreamFileReader(com.hedera.mirror.importer.reader.StreamFileReader) SignatureFileReader(com.hedera.mirror.importer.reader.signature.SignatureFileReader) Collectors.maxBy(java.util.stream.Collectors.maxBy) StreamFile(com.hedera.mirror.common.domain.StreamFile) SignatureVerificationException(com.hedera.mirror.importer.exception.SignatureVerificationException) ExecutionException(java.util.concurrent.ExecutionException) TimeUnit(java.util.concurrent.TimeUnit) MirrorProperties(com.hedera.mirror.importer.MirrorProperties) MeterRegistry(io.micrometer.core.instrument.MeterRegistry) LogManager(org.apache.logging.log4j.LogManager) ArrayList(java.util.ArrayList) Stopwatch(com.google.common.base.Stopwatch) FileStreamSignature(com.hedera.mirror.importer.domain.FileStreamSignature) Callable(java.util.concurrent.Callable) HashMismatchException(com.hedera.mirror.importer.exception.HashMismatchException) InvalidStreamFileException(com.hedera.mirror.importer.exception.InvalidStreamFileException) SignatureVerificationException(com.hedera.mirror.importer.exception.SignatureVerificationException) ExecutionException(java.util.concurrent.ExecutionException) EntityId(com.hedera.mirror.common.domain.entity.EntityId) AtomicInteger(java.util.concurrent.atomic.AtomicInteger) List(java.util.List) ArrayList(java.util.ArrayList)

Example 74 with EntityId

use of com.hedera.mirror.common.domain.entity.EntityId in project hedera-mirror-node by hashgraph.

the class ContractResultServiceImpl method getCreatedContractIds.

@SuppressWarnings("deprecation")
private List<Long> getCreatedContractIds(ContractFunctionResult functionResult, RecordItem recordItem, EntityId parentEntityContractId) {
    List<Long> createdContractIds = new ArrayList<>();
    boolean persist = shouldPersistCreatedContractIDs(recordItem);
    for (ContractID createdContractId : functionResult.getCreatedContractIDsList()) {
        EntityId contractId = entityIdService.lookup(createdContractId);
        if (!EntityId.isEmpty(contractId)) {
            createdContractIds.add(contractId.getId());
            // The parent contract ID can also sometimes appear in the created contract IDs list, so exclude it
            if (persist && !contractId.equals(parentEntityContractId)) {
                processCreatedContractEntity(recordItem, contractId);
            }
        }
    }
    return createdContractIds;
}
Also used : EntityId(com.hedera.mirror.common.domain.entity.EntityId) ArrayList(java.util.ArrayList) ContractID(com.hederahashgraph.api.proto.java.ContractID)

Example 75 with EntityId

use of com.hedera.mirror.common.domain.entity.EntityId in project hedera-mirror-node by hashgraph.

the class TopicMessage method toResponse.

private ConsensusTopicResponse toResponse() {
    var consensusTopicResponseBuilder = ConsensusTopicResponse.newBuilder().setConsensusTimestamp(ProtoUtil.toTimestamp(getConsensusTimestampInstant())).setMessage(ProtoUtil.toByteString(message)).setRunningHash(ProtoUtil.toByteString(runningHash)).setRunningHashVersion(runningHashVersion).setSequenceNumber(sequenceNumber);
    if (getChunkNum() != null) {
        ConsensusMessageChunkInfo.Builder chunkBuilder = ConsensusMessageChunkInfo.newBuilder().setNumber(getChunkNum()).setTotal(getChunkTotal());
        TransactionID transactionID = parseTransactionID(initialTransactionId);
        EntityId payerAccountEntity = EntityId.of(payerAccountId, EntityType.ACCOUNT);
        Instant validStartInstant = LongToInstantConverter.INSTANCE.convert(validStartTimestamp);
        if (transactionID != null) {
            chunkBuilder.setInitialTransactionID(transactionID);
        } else if (payerAccountEntity != null && validStartInstant != null) {
            chunkBuilder.setInitialTransactionID(TransactionID.newBuilder().setAccountID(ProtoUtil.toAccountID(payerAccountEntity)).setTransactionValidStart(ProtoUtil.toTimestamp(validStartInstant)).build());
        }
        consensusTopicResponseBuilder.setChunkInfo(chunkBuilder.build());
    }
    return consensusTopicResponseBuilder.build();
}
Also used : EntityId(com.hedera.mirror.common.domain.entity.EntityId) ConsensusMessageChunkInfo(com.hederahashgraph.api.proto.java.ConsensusMessageChunkInfo) Instant(java.time.Instant) TransactionID(com.hederahashgraph.api.proto.java.TransactionID)

Aggregations

EntityId (com.hedera.mirror.common.domain.entity.EntityId)134 Test (org.junit.jupiter.api.Test)64 ParameterizedTest (org.junit.jupiter.params.ParameterizedTest)43 IntegrationTest (com.hedera.mirror.importer.IntegrationTest)33 Assertions.assertThat (org.assertj.core.api.Assertions.assertThat)22 EntityType (com.hedera.mirror.common.domain.entity.EntityType)21 TokenAccount (com.hedera.mirror.common.domain.token.TokenAccount)21 Token (com.hedera.mirror.common.domain.token.Token)20 Entity (com.hedera.mirror.common.domain.entity.Entity)17 Transaction (com.hedera.mirror.common.domain.transaction.Transaction)17 TransactionBody (com.hederahashgraph.api.proto.java.TransactionBody)16 Assertions (org.junit.jupiter.api.Assertions)16 Contract (com.hedera.mirror.common.domain.contract.Contract)15 ACCOUNT (com.hedera.mirror.common.domain.entity.EntityType.ACCOUNT)15 DomainUtils (com.hedera.mirror.common.util.DomainUtils)15 ByteString (com.google.protobuf.ByteString)14 RecordItem (com.hedera.mirror.common.domain.transaction.RecordItem)14 AccountID (com.hederahashgraph.api.proto.java.AccountID)14 ContractID (com.hederahashgraph.api.proto.java.ContractID)14 Consumer (java.util.function.Consumer)13