Search in sources :

Example 1 with FileStreamSignature

use of com.hedera.mirror.importer.domain.FileStreamSignature in project hedera-mirror-node by hashgraph.

the class SignatureFileReaderV5Test method testReadValidFile.

@Test
void testReadValidFile() {
    StreamFileData streamFileData = StreamFileData.from(signatureFile);
    FileStreamSignature fileStreamSignature = fileReaderV5.read(streamFileData);
    assertNotNull(fileStreamSignature);
    assertThat(fileStreamSignature.getBytes()).isNotEmpty().isEqualTo(streamFileData.getBytes());
    assertArrayEquals(Base64.decodeBase64(entireFileHashBase64.getBytes()), fileStreamSignature.getFileHash());
    assertArrayEquals(Base64.decodeBase64(entireFileSignatureBase64.getBytes()), fileStreamSignature.getFileHashSignature());
    assertArrayEquals(Base64.decodeBase64(metadataHashBase64.getBytes()), fileStreamSignature.getMetadataHash());
    assertArrayEquals(Base64.decodeBase64(metadataSignatureBase64.getBytes()), fileStreamSignature.getMetadataHashSignature());
}
Also used : StreamFileData(com.hedera.mirror.importer.domain.StreamFileData) FileStreamSignature(com.hedera.mirror.importer.domain.FileStreamSignature) Test(org.junit.jupiter.api.Test) DynamicTest(org.junit.jupiter.api.DynamicTest)

Example 2 with FileStreamSignature

use of com.hedera.mirror.importer.domain.FileStreamSignature in project hedera-mirror-node by hashgraph.

the class Downloader method verifySigsAndDownloadDataFiles.

/**
 * For each group of signature files with the same file name: (1) verify that the signature files are signed by
 * corresponding node's PublicKey; (2) For valid signature files, we compare their Hashes to see if at least 1/3 of
 * hashes match. If they do, we download the corresponding data file from a node folder which has valid signature
 * file. (3) compare the hash of data file with Hash which has been agreed on by valid signatures, if match, move
 * the data file into `valid` directory; else download the data file from other valid node folder and compare the
 * hash until we find a match.
 *
 * @param sigFilesMap signature files grouped by filename
 */
private void verifySigsAndDownloadDataFiles(Multimap<String, FileStreamSignature> sigFilesMap) {
    Instant endDate = mirrorProperties.getEndDate();
    for (var sigFilenameIter = sigFilesMap.keySet().iterator(); sigFilenameIter.hasNext(); ) {
        if (ShutdownHelper.isStopping()) {
            return;
        }
        Instant startTime = Instant.now();
        String sigFilename = sigFilenameIter.next();
        Collection<FileStreamSignature> signatures = sigFilesMap.get(sigFilename);
        boolean valid = false;
        try {
            nodeSignatureVerifier.verify(signatures);
        } catch (SignatureVerificationException ex) {
            if (sigFilenameIter.hasNext()) {
                log.warn("Signature verification failed but still have files in the batch, try to process the " + "next group: {}", ex.getMessage());
                continue;
            }
            throw ex;
        }
        for (FileStreamSignature signature : signatures) {
            if (ShutdownHelper.isStopping()) {
                return;
            }
            // Ignore signatures that didn't validate or weren't in the majority
            if (signature.getStatus() != FileStreamSignature.SignatureStatus.CONSENSUS_REACHED) {
                continue;
            }
            try {
                PendingDownload pendingDownload = downloadSignedDataFile(signature);
                if (!pendingDownload.waitForCompletion()) {
                    continue;
                }
                StreamFilename dataFilename = pendingDownload.getStreamFilename();
                StreamFileData streamFileData = new StreamFileData(dataFilename, pendingDownload.getBytes());
                T streamFile = streamFileReader.read(streamFileData);
                streamFile.setNodeAccountId(signature.getNodeAccountId());
                verify(streamFile, signature);
                if (downloaderProperties.isWriteFiles()) {
                    Utility.archiveFile(streamFile.getName(), streamFile.getBytes(), downloaderProperties.getNodeStreamPath(signature.getNodeAccountIdString()));
                }
                if (downloaderProperties.isWriteSignatures()) {
                    signatures.forEach(s -> {
                        Path destination = downloaderProperties.getNodeStreamPath(s.getNodeAccountIdString());
                        Utility.archiveFile(s.getFilename(), s.getBytes(), destination);
                    });
                }
                if (!downloaderProperties.isPersistBytes()) {
                    streamFile.setBytes(null);
                }
                if (dataFilename.getInstant().isAfter(endDate)) {
                    downloaderProperties.setEnabled(false);
                    log.warn("Disabled polling after downloading all files <= endDate ({})", endDate);
                    return;
                }
                onVerified(pendingDownload, streamFile);
                valid = true;
                break;
            } catch (HashMismatchException e) {
                log.warn("Failed to verify data file from node {} corresponding to {}. Will retry another node", signature.getNodeAccountIdString(), sigFilename, e);
            } catch (InterruptedException e) {
                log.warn("Failed to download data file from node {} corresponding to {}", signature.getNodeAccountIdString(), sigFilename, e);
                Thread.currentThread().interrupt();
            } catch (Exception e) {
                log.error("Error downloading data file from node {} corresponding to {}. Will retry another node", signature.getNodeAccountIdString(), sigFilename, e);
            }
        }
        if (!valid) {
            log.error("None of the data files could be verified, signatures: {}", signatures);
        }
        streamVerificationMetric.tag("success", String.valueOf(valid)).register(meterRegistry).record(Duration.between(startTime, Instant.now()));
    }
}
Also used : Path(java.nio.file.Path) StreamFilename(com.hedera.mirror.importer.domain.StreamFilename) Instant(java.time.Instant) FileStreamSignature(com.hedera.mirror.importer.domain.FileStreamSignature) HashMismatchException(com.hedera.mirror.importer.exception.HashMismatchException) HashMismatchException(com.hedera.mirror.importer.exception.HashMismatchException) InvalidStreamFileException(com.hedera.mirror.importer.exception.InvalidStreamFileException) SignatureVerificationException(com.hedera.mirror.importer.exception.SignatureVerificationException) ExecutionException(java.util.concurrent.ExecutionException) StreamFileData(com.hedera.mirror.importer.domain.StreamFileData) SignatureVerificationException(com.hedera.mirror.importer.exception.SignatureVerificationException)

Example 3 with FileStreamSignature

use of com.hedera.mirror.importer.domain.FileStreamSignature in project hedera-mirror-node by hashgraph.

the class Downloader method parseSignatureFile.

private Optional<FileStreamSignature> parseSignatureFile(PendingDownload pendingDownload, EntityId nodeAccountId) throws InterruptedException, ExecutionException {
    String s3Key = pendingDownload.getS3key();
    Stopwatch stopwatch = pendingDownload.getStopwatch();
    if (!pendingDownload.waitForCompletion()) {
        log.warn("Failed downloading {} in {}", s3Key, stopwatch);
        return Optional.empty();
    }
    StreamFilename streamFilename = pendingDownload.getStreamFilename();
    StreamFileData streamFileData = new StreamFileData(streamFilename, pendingDownload.getBytes());
    FileStreamSignature fileStreamSignature = signatureFileReader.read(streamFileData);
    fileStreamSignature.setNodeAccountId(nodeAccountId);
    fileStreamSignature.setStreamType(streamType);
    return Optional.of(fileStreamSignature);
}
Also used : StreamFilename(com.hedera.mirror.importer.domain.StreamFilename) StreamFileData(com.hedera.mirror.importer.domain.StreamFileData) Stopwatch(com.google.common.base.Stopwatch) FileStreamSignature(com.hedera.mirror.importer.domain.FileStreamSignature)

Example 4 with FileStreamSignature

use of com.hedera.mirror.importer.domain.FileStreamSignature in project hedera-mirror-node by hashgraph.

the class NodeSignatureVerifier method statusMap.

private Map<String, Collection<String>> statusMap(Collection<FileStreamSignature> signatures, Map<String, PublicKey> nodeAccountIDPubKeyMap) {
    Map<String, Collection<String>> statusMap = signatures.stream().collect(Collectors.groupingBy(fss -> fss.getStatus().toString(), Collectors.mapping(fss -> fss.getNodeAccountIdString(), Collectors.toCollection(TreeSet::new))));
    Set<String> seenNodes = new HashSet<>();
    signatures.forEach(signature -> seenNodes.add(signature.getNodeAccountId().entityIdToString()));
    Set<String> missingNodes = new TreeSet<>(Sets.difference(nodeAccountIDPubKeyMap.keySet().stream().collect(Collectors.toSet()), seenNodes));
    statusMap.put(SignatureStatus.NOT_FOUND.toString(), missingNodes);
    String streamType = signatures.stream().map(FileStreamSignature::getStreamType).map(StreamType::toString).findFirst().orElse("UNKNOWN");
    for (Map.Entry<String, Collection<String>> entry : statusMap.entrySet()) {
        entry.getValue().forEach(nodeAccountId -> {
            Counter counter = nodeSignatureStatusMetricMap.computeIfAbsent(nodeAccountId, n -> newStatusMetric(nodeAccountId, streamType, entry.getKey()));
            counter.increment();
        });
    }
    // remove CONSENSUS_REACHED for logging purposes
    statusMap.remove(SignatureStatus.CONSENSUS_REACHED.toString());
    return statusMap;
}
Also used : Counter(io.micrometer.core.instrument.Counter) EntityId(com.hedera.mirror.common.domain.entity.EntityId) AddressBook(com.hedera.mirror.common.domain.addressbook.AddressBook) Collection(java.util.Collection) StreamType(com.hedera.mirror.common.domain.StreamType) Signature(java.security.Signature) ConcurrentHashMap(java.util.concurrent.ConcurrentHashMap) Set(java.util.Set) PublicKey(java.security.PublicKey) Multimap(com.google.common.collect.Multimap) Collectors(java.util.stream.Collectors) EntityType(com.hedera.mirror.common.domain.entity.EntityType) SignatureVerificationException(com.hedera.mirror.importer.exception.SignatureVerificationException) Sets(com.google.common.collect.Sets) TreeSet(java.util.TreeSet) HashSet(java.util.HashSet) FileStreamSignature(com.hedera.mirror.importer.domain.FileStreamSignature) AddressBookService(com.hedera.mirror.importer.addressbook.AddressBookService) HashMultimap(com.google.common.collect.HashMultimap) MeterRegistry(io.micrometer.core.instrument.MeterRegistry) Map(java.util.Map) Log4j2(lombok.extern.log4j.Log4j2) SignatureStatus(com.hedera.mirror.importer.domain.FileStreamSignature.SignatureStatus) Named(javax.inject.Named) StreamType(com.hedera.mirror.common.domain.StreamType) Counter(io.micrometer.core.instrument.Counter) TreeSet(java.util.TreeSet) Collection(java.util.Collection) ConcurrentHashMap(java.util.concurrent.ConcurrentHashMap) Map(java.util.Map) HashSet(java.util.HashSet)

Example 5 with FileStreamSignature

use of com.hedera.mirror.importer.domain.FileStreamSignature in project hedera-mirror-node by hashgraph.

the class NodeSignatureVerifierTest method testCannotReachConsensus.

@Test
void testCannotReachConsensus() {
    Map<String, PublicKey> nodeAccountIDPubKeyMap = new HashMap();
    nodeAccountIDPubKeyMap.put("0.0.3", publicKey);
    nodeAccountIDPubKeyMap.put("0.0.4", publicKey);
    nodeAccountIDPubKeyMap.put("0.0.5", publicKey);
    nodeAccountIDPubKeyMap.put("0.0.6", publicKey);
    when(currentAddressBook.getNodeAccountIDPubKeyMap()).thenReturn(nodeAccountIDPubKeyMap);
    List<FileStreamSignature> fileStreamSignatures = Arrays.asList(buildBareBonesFileStreamSignature());
    Exception e = assertThrows(SignatureVerificationException.class, () -> nodeSignatureVerifier.verify(fileStreamSignatures));
    assertTrue(e.getMessage().contains("Insufficient downloaded signature file count, requires at least 0.333"));
}
Also used : HashMap(java.util.HashMap) PublicKey(java.security.PublicKey) FileStreamSignature(com.hedera.mirror.importer.domain.FileStreamSignature) GeneralSecurityException(java.security.GeneralSecurityException) SignatureVerificationException(com.hedera.mirror.importer.exception.SignatureVerificationException) NoSuchAlgorithmException(java.security.NoSuchAlgorithmException) Test(org.junit.jupiter.api.Test)

Aggregations

FileStreamSignature (com.hedera.mirror.importer.domain.FileStreamSignature)24 Test (org.junit.jupiter.api.Test)14 SignatureVerificationException (com.hedera.mirror.importer.exception.SignatureVerificationException)12 GeneralSecurityException (java.security.GeneralSecurityException)7 NoSuchAlgorithmException (java.security.NoSuchAlgorithmException)7 PublicKey (java.security.PublicKey)7 StreamFileData (com.hedera.mirror.importer.domain.StreamFileData)5 EntityId (com.hedera.mirror.common.domain.entity.EntityId)4 InvalidStreamFileException (com.hedera.mirror.importer.exception.InvalidStreamFileException)4 HashMap (java.util.HashMap)4 AddressBook (com.hedera.mirror.common.domain.addressbook.AddressBook)3 StreamFilename (com.hedera.mirror.importer.domain.StreamFilename)3 Stopwatch (com.google.common.base.Stopwatch)2 Multimap (com.google.common.collect.Multimap)2 StreamType (com.hedera.mirror.common.domain.StreamType)2 AddressBookService (com.hedera.mirror.importer.addressbook.AddressBookService)2 HashMismatchException (com.hedera.mirror.importer.exception.HashMismatchException)2 SignatureFileParsingException (com.hedera.mirror.importer.exception.SignatureFileParsingException)2 ValidatedDataInputStream (com.hedera.mirror.importer.reader.ValidatedDataInputStream)2 MeterRegistry (io.micrometer.core.instrument.MeterRegistry)2