Search in sources :

Example 21 with FileStreamSignature

use of com.hedera.mirror.importer.domain.FileStreamSignature in project hedera-mirror-node by hashgraph.

the class Downloader method downloadAndParseSigFiles.

/**
 * Download and parse all signature files with a timestamp later than the last valid file. Put signature files into
 * a multi-map sorted and grouped by the timestamp.
 *
 * @param addressBook the current address book
 * @return a multi-map of signature file objects from different nodes, grouped by filename
 */
private Multimap<String, FileStreamSignature> downloadAndParseSigFiles(AddressBook addressBook) throws InterruptedException {
    String startAfterFilename = getStartAfterFilename();
    Multimap<String, FileStreamSignature> sigFilesMap = Multimaps.synchronizedSortedSetMultimap(TreeMultimap.create());
    Set<EntityId> nodeAccountIds = addressBook.getNodeSet();
    List<Callable<Object>> tasks = new ArrayList<>(nodeAccountIds.size());
    AtomicInteger totalDownloads = new AtomicInteger();
    log.info("Downloading signature files created after file: {}", startAfterFilename);
    /*
         * For each node, create a thread that will make S3 ListObject requests as many times as necessary to
         * start maxDownloads download operations.
         */
    for (EntityId nodeAccountId : nodeAccountIds) {
        tasks.add(Executors.callable(() -> {
            String nodeAccountIdStr = nodeAccountId.entityIdToString();
            Stopwatch stopwatch = Stopwatch.createStarted();
            try {
                List<S3Object> s3Objects = listFiles(startAfterFilename, nodeAccountIdStr);
                List<PendingDownload> pendingDownloads = downloadSignatureFiles(nodeAccountIdStr, s3Objects);
                AtomicInteger count = new AtomicInteger();
                pendingDownloads.forEach(pendingDownload -> {
                    try {
                        parseSignatureFile(pendingDownload, nodeAccountId).ifPresent(fileStreamSignature -> {
                            sigFilesMap.put(fileStreamSignature.getFilename(), fileStreamSignature);
                            count.incrementAndGet();
                            totalDownloads.incrementAndGet();
                        });
                    } catch (InterruptedException ex) {
                        log.warn("Failed downloading {} in {}", pendingDownload.getS3key(), pendingDownload.getStopwatch(), ex);
                        Thread.currentThread().interrupt();
                    } catch (Exception ex) {
                        log.warn("Failed to parse signature file {}: {}", pendingDownload.getS3key(), ex);
                    }
                });
                if (count.get() > 0) {
                    log.info("Downloaded {} signatures for node {} in {}", count.get(), nodeAccountIdStr, stopwatch);
                }
            } catch (InterruptedException e) {
                log.error("Error downloading signature files for node {} after {}", nodeAccountIdStr, stopwatch, e);
                Thread.currentThread().interrupt();
            } catch (Exception e) {
                log.error("Error downloading signature files for node {} after {}", nodeAccountIdStr, stopwatch, e);
            }
        }));
    }
    // Wait for all tasks to complete.
    // invokeAll() does return Futures, but it waits for all to complete (so they're returned in a completed state).
    Stopwatch stopwatch = Stopwatch.createStarted();
    signatureDownloadThreadPool.invokeAll(tasks);
    if (totalDownloads.get() > 0) {
        var rate = (int) (1000000.0 * totalDownloads.get() / stopwatch.elapsed(TimeUnit.MICROSECONDS));
        log.info("Downloaded {} signatures in {} ({}/s)", totalDownloads, stopwatch, rate);
    }
    return sigFilesMap;
}
Also used : EntityId(com.hedera.mirror.common.domain.entity.EntityId) AddressBook(com.hedera.mirror.common.domain.addressbook.AddressBook) FileStreamSignature(com.hedera.mirror.importer.domain.FileStreamSignature) TreeMultimap(com.google.common.collect.TreeMultimap) AtomicInteger(java.util.concurrent.atomic.AtomicInteger) Duration(java.time.Duration) Map(java.util.Map) GetObjectRequest(software.amazon.awssdk.services.s3.model.GetObjectRequest) AsyncResponseTransformer(software.amazon.awssdk.core.async.AsyncResponseTransformer) Path(java.nio.file.Path) Utility(com.hedera.mirror.importer.util.Utility) ListObjectsRequest(software.amazon.awssdk.services.s3.model.ListObjectsRequest) S3AsyncClient(software.amazon.awssdk.services.s3.S3AsyncClient) SIGNATURE(com.hedera.mirror.importer.domain.StreamFilename.FileType.SIGNATURE) Collection(java.util.Collection) StreamType(com.hedera.mirror.common.domain.StreamType) Set(java.util.Set) HashMismatchException(com.hedera.mirror.importer.exception.HashMismatchException) Instant(java.time.Instant) Collectors(java.util.stream.Collectors) Executors(java.util.concurrent.Executors) Objects(java.util.Objects) InvalidStreamFileException(com.hedera.mirror.importer.exception.InvalidStreamFileException) List(java.util.List) Logger(org.apache.logging.log4j.Logger) Optional(java.util.Optional) SHA384(com.hedera.mirror.common.domain.DigestAlgorithm.SHA384) ShutdownHelper(com.hedera.mirror.importer.util.ShutdownHelper) StreamFileData(com.hedera.mirror.importer.domain.StreamFileData) Stopwatch(com.google.common.base.Stopwatch) S3Object(software.amazon.awssdk.services.s3.model.S3Object) Collectors.groupingBy(java.util.stream.Collectors.groupingBy) Callable(java.util.concurrent.Callable) Multimap(com.google.common.collect.Multimap) AtomicReference(java.util.concurrent.atomic.AtomicReference) Multimaps(com.google.common.collect.Multimaps) ArrayList(java.util.ArrayList) RequestPayer(software.amazon.awssdk.services.s3.model.RequestPayer) AddressBookService(com.hedera.mirror.importer.addressbook.AddressBookService) Timer(io.micrometer.core.instrument.Timer) StreamFilename(com.hedera.mirror.importer.domain.StreamFilename) ExecutorService(java.util.concurrent.ExecutorService) MirrorDateRangePropertiesProcessor(com.hedera.mirror.importer.config.MirrorDateRangePropertiesProcessor) StreamFileReader(com.hedera.mirror.importer.reader.StreamFileReader) SignatureFileReader(com.hedera.mirror.importer.reader.signature.SignatureFileReader) Collectors.maxBy(java.util.stream.Collectors.maxBy) StreamFile(com.hedera.mirror.common.domain.StreamFile) SignatureVerificationException(com.hedera.mirror.importer.exception.SignatureVerificationException) ExecutionException(java.util.concurrent.ExecutionException) TimeUnit(java.util.concurrent.TimeUnit) MirrorProperties(com.hedera.mirror.importer.MirrorProperties) MeterRegistry(io.micrometer.core.instrument.MeterRegistry) LogManager(org.apache.logging.log4j.LogManager) ArrayList(java.util.ArrayList) Stopwatch(com.google.common.base.Stopwatch) FileStreamSignature(com.hedera.mirror.importer.domain.FileStreamSignature) Callable(java.util.concurrent.Callable) HashMismatchException(com.hedera.mirror.importer.exception.HashMismatchException) InvalidStreamFileException(com.hedera.mirror.importer.exception.InvalidStreamFileException) SignatureVerificationException(com.hedera.mirror.importer.exception.SignatureVerificationException) ExecutionException(java.util.concurrent.ExecutionException) EntityId(com.hedera.mirror.common.domain.entity.EntityId) AtomicInteger(java.util.concurrent.atomic.AtomicInteger) List(java.util.List) ArrayList(java.util.ArrayList)

Example 22 with FileStreamSignature

use of com.hedera.mirror.importer.domain.FileStreamSignature in project hedera-mirror-node by hashgraph.

the class NodeSignatureVerifier method verifySignature.

/**
 * check whether the given signature is valid
 *
 * @param fileStreamSignature    the data that was signed
 * @param nodeAccountIDPubKeyMap map of node account ids (as Strings) and their public keys
 * @return true if the signature is valid
 */
private boolean verifySignature(FileStreamSignature fileStreamSignature, Map<String, PublicKey> nodeAccountIDPubKeyMap) {
    PublicKey publicKey = nodeAccountIDPubKeyMap.get(fileStreamSignature.getNodeAccountIdString());
    if (publicKey == null) {
        log.warn("Missing PublicKey for node {}", fileStreamSignature.getNodeAccountIdString());
        return false;
    }
    if (fileStreamSignature.getFileHashSignature() == null) {
        log.error("Missing signature data: {}", fileStreamSignature);
        return false;
    }
    try {
        log.trace("Verifying signature: {}", fileStreamSignature);
        Signature sig = Signature.getInstance(fileStreamSignature.getSignatureType().getAlgorithm(), fileStreamSignature.getSignatureType().getProvider());
        sig.initVerify(publicKey);
        sig.update(fileStreamSignature.getFileHash());
        if (!sig.verify(fileStreamSignature.getFileHashSignature())) {
            return false;
        }
        if (fileStreamSignature.getMetadataHashSignature() != null) {
            sig.update(fileStreamSignature.getMetadataHash());
            return sig.verify(fileStreamSignature.getMetadataHashSignature());
        }
        return true;
    } catch (Exception e) {
        log.error("Failed to verify signature with public key {}: {}", publicKey, fileStreamSignature, e);
    }
    return false;
}
Also used : PublicKey(java.security.PublicKey) Signature(java.security.Signature) FileStreamSignature(com.hedera.mirror.importer.domain.FileStreamSignature) SignatureVerificationException(com.hedera.mirror.importer.exception.SignatureVerificationException)

Example 23 with FileStreamSignature

use of com.hedera.mirror.importer.domain.FileStreamSignature in project hedera-mirror-node by hashgraph.

the class NodeSignatureVerifier method verify.

/**
 * Verifies that the signature files satisfy the consensus requirement:
 * <ol>
 *  <li>At least 1/3 signature files are present</li>
 *  <li>For a signature file, we validate it by checking if it's signed by corresponding node's PublicKey. For valid
 *      signature files, we compare their hashes to see if at least 1/3 have hashes that match. If a signature is
 *      valid, we put the hash in its content and its file to the map, to see if at least 1/3 valid signatures have
 *      the same hash</li>
 * </ol>
 *
 * @param signatures a list of signature files which have the same filename
 * @throws SignatureVerificationException
 */
public void verify(Collection<FileStreamSignature> signatures) throws SignatureVerificationException {
    AddressBook currentAddressBook = addressBookService.getCurrent();
    Map<String, PublicKey> nodeAccountIDPubKeyMap = currentAddressBook.getNodeAccountIDPubKeyMap();
    Multimap<String, FileStreamSignature> signatureHashMap = HashMultimap.create();
    String filename = signatures.stream().map(FileStreamSignature::getFilename).findFirst().orElse("unknown");
    int consensusCount = 0;
    long sigFileCount = signatures.size();
    long nodeCount = nodeAccountIDPubKeyMap.size();
    if (!canReachConsensus(sigFileCount, nodeCount)) {
        throw new SignatureVerificationException(String.format("Insufficient downloaded signature file count, requires at least %.03f to reach consensus, got %d" + " out of %d for file %s: %s", commonDownloaderProperties.getConsensusRatio(), sigFileCount, nodeCount, filename, statusMap(signatures, nodeAccountIDPubKeyMap)));
    }
    for (FileStreamSignature fileStreamSignature : signatures) {
        if (verifySignature(fileStreamSignature, nodeAccountIDPubKeyMap)) {
            fileStreamSignature.setStatus(SignatureStatus.VERIFIED);
            signatureHashMap.put(fileStreamSignature.getFileHashAsHex(), fileStreamSignature);
        }
    }
    if (commonDownloaderProperties.getConsensusRatio() == 0 && signatureHashMap.size() > 0) {
        log.debug("Signature file {} does not require consensus, skipping consensus check", filename);
        return;
    }
    for (String key : signatureHashMap.keySet()) {
        Collection<FileStreamSignature> validatedSignatures = signatureHashMap.get(key);
        if (canReachConsensus(validatedSignatures.size(), nodeCount)) {
            consensusCount += validatedSignatures.size();
            validatedSignatures.forEach(s -> s.setStatus(SignatureStatus.CONSENSUS_REACHED));
        }
    }
    if (consensusCount == nodeCount) {
        log.debug("Verified signature file {} reached consensus", filename);
        return;
    } else if (consensusCount > 0) {
        log.warn("Verified signature file {} reached consensus but with some errors: {}", filename, statusMap(signatures, nodeAccountIDPubKeyMap));
        return;
    }
    throw new SignatureVerificationException("Signature verification failed for file " + filename + ": " + statusMap(signatures, nodeAccountIDPubKeyMap));
}
Also used : AddressBook(com.hedera.mirror.common.domain.addressbook.AddressBook) PublicKey(java.security.PublicKey) SignatureVerificationException(com.hedera.mirror.importer.exception.SignatureVerificationException) FileStreamSignature(com.hedera.mirror.importer.domain.FileStreamSignature)

Example 24 with FileStreamSignature

use of com.hedera.mirror.importer.domain.FileStreamSignature in project hedera-mirror-node by hashgraph.

the class SignatureFileReaderV2Test method testReadValidFile.

@Test
void testReadValidFile() {
    StreamFileData streamFileData = StreamFileData.from(signatureFile);
    FileStreamSignature fileStreamSignature = fileReaderV2.read(streamFileData);
    assertNotNull(fileStreamSignature);
    assertThat(fileStreamSignature.getBytes()).isNotEmpty().isEqualTo(streamFileData.getBytes());
    assertArrayEquals(Base64.decodeBase64(entireFileHashBase64.getBytes()), fileStreamSignature.getFileHash());
    assertArrayEquals(Base64.decodeBase64(entireFileSignatureBase64.getBytes()), fileStreamSignature.getFileHashSignature());
}
Also used : StreamFileData(com.hedera.mirror.importer.domain.StreamFileData) FileStreamSignature(com.hedera.mirror.importer.domain.FileStreamSignature) Test(org.junit.jupiter.api.Test) DynamicTest(org.junit.jupiter.api.DynamicTest)

Aggregations

FileStreamSignature (com.hedera.mirror.importer.domain.FileStreamSignature)24 Test (org.junit.jupiter.api.Test)14 SignatureVerificationException (com.hedera.mirror.importer.exception.SignatureVerificationException)12 GeneralSecurityException (java.security.GeneralSecurityException)7 NoSuchAlgorithmException (java.security.NoSuchAlgorithmException)7 PublicKey (java.security.PublicKey)7 StreamFileData (com.hedera.mirror.importer.domain.StreamFileData)5 EntityId (com.hedera.mirror.common.domain.entity.EntityId)4 InvalidStreamFileException (com.hedera.mirror.importer.exception.InvalidStreamFileException)4 HashMap (java.util.HashMap)4 AddressBook (com.hedera.mirror.common.domain.addressbook.AddressBook)3 StreamFilename (com.hedera.mirror.importer.domain.StreamFilename)3 Stopwatch (com.google.common.base.Stopwatch)2 Multimap (com.google.common.collect.Multimap)2 StreamType (com.hedera.mirror.common.domain.StreamType)2 AddressBookService (com.hedera.mirror.importer.addressbook.AddressBookService)2 HashMismatchException (com.hedera.mirror.importer.exception.HashMismatchException)2 SignatureFileParsingException (com.hedera.mirror.importer.exception.SignatureFileParsingException)2 ValidatedDataInputStream (com.hedera.mirror.importer.reader.ValidatedDataInputStream)2 MeterRegistry (io.micrometer.core.instrument.MeterRegistry)2