use of com.hedera.mirror.importer.domain.FileStreamSignature in project hedera-mirror-node by hashgraph.
the class Downloader method downloadAndParseSigFiles.
/**
* Download and parse all signature files with a timestamp later than the last valid file. Put signature files into
* a multi-map sorted and grouped by the timestamp.
*
* @param addressBook the current address book
* @return a multi-map of signature file objects from different nodes, grouped by filename
*/
private Multimap<String, FileStreamSignature> downloadAndParseSigFiles(AddressBook addressBook) throws InterruptedException {
String startAfterFilename = getStartAfterFilename();
Multimap<String, FileStreamSignature> sigFilesMap = Multimaps.synchronizedSortedSetMultimap(TreeMultimap.create());
Set<EntityId> nodeAccountIds = addressBook.getNodeSet();
List<Callable<Object>> tasks = new ArrayList<>(nodeAccountIds.size());
AtomicInteger totalDownloads = new AtomicInteger();
log.info("Downloading signature files created after file: {}", startAfterFilename);
/*
* For each node, create a thread that will make S3 ListObject requests as many times as necessary to
* start maxDownloads download operations.
*/
for (EntityId nodeAccountId : nodeAccountIds) {
tasks.add(Executors.callable(() -> {
String nodeAccountIdStr = nodeAccountId.entityIdToString();
Stopwatch stopwatch = Stopwatch.createStarted();
try {
List<S3Object> s3Objects = listFiles(startAfterFilename, nodeAccountIdStr);
List<PendingDownload> pendingDownloads = downloadSignatureFiles(nodeAccountIdStr, s3Objects);
AtomicInteger count = new AtomicInteger();
pendingDownloads.forEach(pendingDownload -> {
try {
parseSignatureFile(pendingDownload, nodeAccountId).ifPresent(fileStreamSignature -> {
sigFilesMap.put(fileStreamSignature.getFilename(), fileStreamSignature);
count.incrementAndGet();
totalDownloads.incrementAndGet();
});
} catch (InterruptedException ex) {
log.warn("Failed downloading {} in {}", pendingDownload.getS3key(), pendingDownload.getStopwatch(), ex);
Thread.currentThread().interrupt();
} catch (Exception ex) {
log.warn("Failed to parse signature file {}: {}", pendingDownload.getS3key(), ex);
}
});
if (count.get() > 0) {
log.info("Downloaded {} signatures for node {} in {}", count.get(), nodeAccountIdStr, stopwatch);
}
} catch (InterruptedException e) {
log.error("Error downloading signature files for node {} after {}", nodeAccountIdStr, stopwatch, e);
Thread.currentThread().interrupt();
} catch (Exception e) {
log.error("Error downloading signature files for node {} after {}", nodeAccountIdStr, stopwatch, e);
}
}));
}
// Wait for all tasks to complete.
// invokeAll() does return Futures, but it waits for all to complete (so they're returned in a completed state).
Stopwatch stopwatch = Stopwatch.createStarted();
signatureDownloadThreadPool.invokeAll(tasks);
if (totalDownloads.get() > 0) {
var rate = (int) (1000000.0 * totalDownloads.get() / stopwatch.elapsed(TimeUnit.MICROSECONDS));
log.info("Downloaded {} signatures in {} ({}/s)", totalDownloads, stopwatch, rate);
}
return sigFilesMap;
}
use of com.hedera.mirror.importer.domain.FileStreamSignature in project hedera-mirror-node by hashgraph.
the class NodeSignatureVerifier method verifySignature.
/**
* check whether the given signature is valid
*
* @param fileStreamSignature the data that was signed
* @param nodeAccountIDPubKeyMap map of node account ids (as Strings) and their public keys
* @return true if the signature is valid
*/
private boolean verifySignature(FileStreamSignature fileStreamSignature, Map<String, PublicKey> nodeAccountIDPubKeyMap) {
PublicKey publicKey = nodeAccountIDPubKeyMap.get(fileStreamSignature.getNodeAccountIdString());
if (publicKey == null) {
log.warn("Missing PublicKey for node {}", fileStreamSignature.getNodeAccountIdString());
return false;
}
if (fileStreamSignature.getFileHashSignature() == null) {
log.error("Missing signature data: {}", fileStreamSignature);
return false;
}
try {
log.trace("Verifying signature: {}", fileStreamSignature);
Signature sig = Signature.getInstance(fileStreamSignature.getSignatureType().getAlgorithm(), fileStreamSignature.getSignatureType().getProvider());
sig.initVerify(publicKey);
sig.update(fileStreamSignature.getFileHash());
if (!sig.verify(fileStreamSignature.getFileHashSignature())) {
return false;
}
if (fileStreamSignature.getMetadataHashSignature() != null) {
sig.update(fileStreamSignature.getMetadataHash());
return sig.verify(fileStreamSignature.getMetadataHashSignature());
}
return true;
} catch (Exception e) {
log.error("Failed to verify signature with public key {}: {}", publicKey, fileStreamSignature, e);
}
return false;
}
use of com.hedera.mirror.importer.domain.FileStreamSignature in project hedera-mirror-node by hashgraph.
the class NodeSignatureVerifier method verify.
/**
* Verifies that the signature files satisfy the consensus requirement:
* <ol>
* <li>At least 1/3 signature files are present</li>
* <li>For a signature file, we validate it by checking if it's signed by corresponding node's PublicKey. For valid
* signature files, we compare their hashes to see if at least 1/3 have hashes that match. If a signature is
* valid, we put the hash in its content and its file to the map, to see if at least 1/3 valid signatures have
* the same hash</li>
* </ol>
*
* @param signatures a list of signature files which have the same filename
* @throws SignatureVerificationException
*/
public void verify(Collection<FileStreamSignature> signatures) throws SignatureVerificationException {
AddressBook currentAddressBook = addressBookService.getCurrent();
Map<String, PublicKey> nodeAccountIDPubKeyMap = currentAddressBook.getNodeAccountIDPubKeyMap();
Multimap<String, FileStreamSignature> signatureHashMap = HashMultimap.create();
String filename = signatures.stream().map(FileStreamSignature::getFilename).findFirst().orElse("unknown");
int consensusCount = 0;
long sigFileCount = signatures.size();
long nodeCount = nodeAccountIDPubKeyMap.size();
if (!canReachConsensus(sigFileCount, nodeCount)) {
throw new SignatureVerificationException(String.format("Insufficient downloaded signature file count, requires at least %.03f to reach consensus, got %d" + " out of %d for file %s: %s", commonDownloaderProperties.getConsensusRatio(), sigFileCount, nodeCount, filename, statusMap(signatures, nodeAccountIDPubKeyMap)));
}
for (FileStreamSignature fileStreamSignature : signatures) {
if (verifySignature(fileStreamSignature, nodeAccountIDPubKeyMap)) {
fileStreamSignature.setStatus(SignatureStatus.VERIFIED);
signatureHashMap.put(fileStreamSignature.getFileHashAsHex(), fileStreamSignature);
}
}
if (commonDownloaderProperties.getConsensusRatio() == 0 && signatureHashMap.size() > 0) {
log.debug("Signature file {} does not require consensus, skipping consensus check", filename);
return;
}
for (String key : signatureHashMap.keySet()) {
Collection<FileStreamSignature> validatedSignatures = signatureHashMap.get(key);
if (canReachConsensus(validatedSignatures.size(), nodeCount)) {
consensusCount += validatedSignatures.size();
validatedSignatures.forEach(s -> s.setStatus(SignatureStatus.CONSENSUS_REACHED));
}
}
if (consensusCount == nodeCount) {
log.debug("Verified signature file {} reached consensus", filename);
return;
} else if (consensusCount > 0) {
log.warn("Verified signature file {} reached consensus but with some errors: {}", filename, statusMap(signatures, nodeAccountIDPubKeyMap));
return;
}
throw new SignatureVerificationException("Signature verification failed for file " + filename + ": " + statusMap(signatures, nodeAccountIDPubKeyMap));
}
use of com.hedera.mirror.importer.domain.FileStreamSignature in project hedera-mirror-node by hashgraph.
the class SignatureFileReaderV2Test method testReadValidFile.
@Test
void testReadValidFile() {
StreamFileData streamFileData = StreamFileData.from(signatureFile);
FileStreamSignature fileStreamSignature = fileReaderV2.read(streamFileData);
assertNotNull(fileStreamSignature);
assertThat(fileStreamSignature.getBytes()).isNotEmpty().isEqualTo(streamFileData.getBytes());
assertArrayEquals(Base64.decodeBase64(entireFileHashBase64.getBytes()), fileStreamSignature.getFileHash());
assertArrayEquals(Base64.decodeBase64(entireFileSignatureBase64.getBytes()), fileStreamSignature.getFileHashSignature());
}
Aggregations