Examples with SIGNATURE com.hedera.mirror.importer.domain.StreamFilename.FileType.SIGNATURE used on opensource projects

Example 1 with SIGNATURE

use of com.hedera.mirror.importer.domain.StreamFilename.FileType.SIGNATURE in project hedera-mirror-node by hashgraph.

the class Downloader method downloadAndParseSigFiles.

/**
 * Download and parse all signature files with a timestamp later than the last valid file. Put signature files into
 * a multi-map sorted and grouped by the timestamp.
 *
 * @param addressBook the current address book
 * @return a multi-map of signature file objects from different nodes, grouped by filename
 */
private Multimap<String, FileStreamSignature> downloadAndParseSigFiles(AddressBook addressBook) throws InterruptedException {
    String startAfterFilename = getStartAfterFilename();
    Multimap<String, FileStreamSignature> sigFilesMap = Multimaps.synchronizedSortedSetMultimap(TreeMultimap.create());
    Set<EntityId> nodeAccountIds = addressBook.getNodeSet();
    List<Callable<Object>> tasks = new ArrayList<>(nodeAccountIds.size());
    AtomicInteger totalDownloads = new AtomicInteger();
    log.info("Downloading signature files created after file: {}", startAfterFilename);
    /*
         * For each node, create a thread that will make S3 ListObject requests as many times as necessary to
         * start maxDownloads download operations.
         */
    for (EntityId nodeAccountId : nodeAccountIds) {
        tasks.add(Executors.callable(() -> {
            String nodeAccountIdStr = nodeAccountId.entityIdToString();
            Stopwatch stopwatch = Stopwatch.createStarted();
            try {
                List<S3Object> s3Objects = listFiles(startAfterFilename, nodeAccountIdStr);
                List<PendingDownload> pendingDownloads = downloadSignatureFiles(nodeAccountIdStr, s3Objects);
                AtomicInteger count = new AtomicInteger();
                pendingDownloads.forEach(pendingDownload -> {
                    try {
                        parseSignatureFile(pendingDownload, nodeAccountId).ifPresent(fileStreamSignature -> {
                            sigFilesMap.put(fileStreamSignature.getFilename(), fileStreamSignature);
                            count.incrementAndGet();
                            totalDownloads.incrementAndGet();
                        });
                    } catch (InterruptedException ex) {
                        log.warn("Failed downloading {} in {}", pendingDownload.getS3key(), pendingDownload.getStopwatch(), ex);
                        Thread.currentThread().interrupt();
                    } catch (Exception ex) {
                        log.warn("Failed to parse signature file {}: {}", pendingDownload.getS3key(), ex);
                    }
                });
                if (count.get() > 0) {
                    log.info("Downloaded {} signatures for node {} in {}", count.get(), nodeAccountIdStr, stopwatch);
                }
            } catch (InterruptedException e) {
                log.error("Error downloading signature files for node {} after {}", nodeAccountIdStr, stopwatch, e);
                Thread.currentThread().interrupt();
            } catch (Exception e) {
                log.error("Error downloading signature files for node {} after {}", nodeAccountIdStr, stopwatch, e);
            }
        }));
    }
    // Wait for all tasks to complete.
    // invokeAll() does return Futures, but it waits for all to complete (so they're returned in a completed state).
    Stopwatch stopwatch = Stopwatch.createStarted();
    signatureDownloadThreadPool.invokeAll(tasks);
    if (totalDownloads.get() > 0) {
        var rate = (int) (1000000.0 * totalDownloads.get() / stopwatch.elapsed(TimeUnit.MICROSECONDS));
        log.info("Downloaded {} signatures in {} ({}/s)", totalDownloads, stopwatch, rate);
    }
    return sigFilesMap;
}