use of com.hedera.mirror.common.domain.StreamType in project hedera-mirror-node by hashgraph.
the class StreamFilename method extractTypeInfo.
private static TypeInfo extractTypeInfo(String filename) {
List<String> parts = FILENAME_SPLITTER.splitToList(filename);
if (parts.size() < 2) {
throw new InvalidStreamFileException("Failed to determine StreamType for filename: " + filename);
}
String last = parts.get(parts.size() - 1);
String secondLast = parts.get(parts.size() - 2);
for (StreamType type : StreamType.values()) {
String suffix = type.getSuffix();
if (!StringUtils.isEmpty(suffix) && !filename.contains(suffix)) {
continue;
}
Map<String, StreamType.Extension> extensions = STREAM_TYPE_EXTENSION_MAP.get(type);
if (extensions.containsKey(last)) {
// if last matches extension, the file is not compressed
FileType fileType = last.endsWith(StreamType.SIGNATURE_SUFFIX) ? SIGNATURE : DATA;
return TypeInfo.of(null, extensions.get(last), fileType, type);
}
if (extensions.containsKey(secondLast)) {
// otherwise if secondLast matches extension, last is the compression extension
FileType fileType = secondLast.endsWith(StreamType.SIGNATURE_SUFFIX) ? SIGNATURE : DATA;
return TypeInfo.of(last, extensions.get(secondLast), fileType, type);
}
}
throw new InvalidStreamFileException("Failed to determine StreamType for filename: " + filename);
}
use of com.hedera.mirror.common.domain.StreamType in project hedera-mirror-node by hashgraph.
the class NodeSignatureVerifier method statusMap.
private Map<String, Collection<String>> statusMap(Collection<FileStreamSignature> signatures, Map<String, PublicKey> nodeAccountIDPubKeyMap) {
Map<String, Collection<String>> statusMap = signatures.stream().collect(Collectors.groupingBy(fss -> fss.getStatus().toString(), Collectors.mapping(fss -> fss.getNodeAccountIdString(), Collectors.toCollection(TreeSet::new))));
Set<String> seenNodes = new HashSet<>();
signatures.forEach(signature -> seenNodes.add(signature.getNodeAccountId().entityIdToString()));
Set<String> missingNodes = new TreeSet<>(Sets.difference(nodeAccountIDPubKeyMap.keySet().stream().collect(Collectors.toSet()), seenNodes));
statusMap.put(SignatureStatus.NOT_FOUND.toString(), missingNodes);
String streamType = signatures.stream().map(FileStreamSignature::getStreamType).map(StreamType::toString).findFirst().orElse("UNKNOWN");
for (Map.Entry<String, Collection<String>> entry : statusMap.entrySet()) {
entry.getValue().forEach(nodeAccountId -> {
Counter counter = nodeSignatureStatusMetricMap.computeIfAbsent(nodeAccountId, n -> newStatusMetric(nodeAccountId, streamType, entry.getKey()));
counter.increment();
});
}
// remove CONSENSUS_REACHED for logging purposes
statusMap.remove(SignatureStatus.CONSENSUS_REACHED.toString());
return statusMap;
}
use of com.hedera.mirror.common.domain.StreamType in project hedera-mirror-node by hashgraph.
the class MirrorDateRangePropertiesProcessor method getLastStreamFile.
/**
* Gets the latest stream file for downloader based on startDate in MirrorProperties, the startDateAdjustment and
* last valid downloaded stream file.
*
* @param streamType What type of stream to retrieve
* @return The latest stream file from the database or a dummy stream file if it calculated a different effective
* start date
*/
public <T extends StreamFile> Optional<T> getLastStreamFile(StreamType streamType) {
Instant startDate = mirrorProperties.getStartDate();
Optional<T> streamFile = findLatest(streamType);
Instant lastFileInstant = streamFile.map(StreamFile::getConsensusStart).map(nanos -> Instant.ofEpochSecond(0, nanos)).orElse(null);
Instant effectiveStartDate = STARTUP_TIME;
boolean hasStreamFile = lastFileInstant != null;
if (startDate != null) {
effectiveStartDate = max(startDate, hasStreamFile ? lastFileInstant : Instant.EPOCH);
} else if (hasStreamFile) {
effectiveStartDate = lastFileInstant;
} else if (mirrorProperties.getNetwork() == MirrorProperties.HederaNetwork.DEMO) {
// Demo network contains only data in the past, so don't default to now
effectiveStartDate = Instant.EPOCH;
}
Instant endDate = mirrorProperties.getEndDate();
if (startDate != null && startDate.compareTo(endDate) > 0) {
throw new InvalidConfigurationException(String.format("Date range constraint violation: " + "startDate (%s) > endDate (%s)", startDate, endDate));
}
if (effectiveStartDate.compareTo(endDate) > 0) {
throw new InvalidConfigurationException(String.format("Date range constraint violation for %s downloader: effective startDate (%s) > endDate (%s)", streamType, effectiveStartDate, endDate));
}
if (!effectiveStartDate.equals(lastFileInstant)) {
Instant verifyHashAfter = mirrorProperties.getVerifyHashAfter();
if (verifyHashAfter == null || verifyHashAfter.isBefore(effectiveStartDate)) {
mirrorProperties.setVerifyHashAfter(effectiveStartDate);
log.debug("Set verifyHashAfter to {}", effectiveStartDate);
}
String filename = StreamFilename.getFilename(streamType, DATA, effectiveStartDate);
T effectiveStreamFile = (T) ReflectUtils.newInstance(streamType.getStreamFileClass());
effectiveStreamFile.setConsensusStart(DomainUtils.convertToNanosMax(effectiveStartDate));
effectiveStreamFile.setName(filename);
effectiveStreamFile.setIndex(streamFile.map(StreamFile::getIndex).orElse(null));
streamFile = Optional.of(effectiveStreamFile);
}
log.info("{}: downloader will download files in time range ({}, {}]", streamType, effectiveStartDate, mirrorProperties.getEndDate());
return streamFile;
}
use of com.hedera.mirror.common.domain.StreamType in project hedera-mirror-node by hashgraph.
the class MirrorDateRangePropertiesProcessor method newDateRangeFilter.
private DateRangeFilter newDateRangeFilter(StreamType streamType) {
DownloaderProperties downloaderProperties = getDownloaderProperties(streamType);
if (!downloaderProperties.isEnabled()) {
return DateRangeFilter.empty();
}
Instant startDate = mirrorProperties.getStartDate();
Instant endDate = mirrorProperties.getEndDate();
Instant lastFileInstant = findLatest(streamType).map(StreamFile::getConsensusStart).map(nanos -> Instant.ofEpochSecond(0, nanos)).orElse(null);
Instant filterStartDate = lastFileInstant;
if (startDate != null && startDate.compareTo(endDate) > 0) {
throw new InvalidConfigurationException(String.format("Date range constraint violation: " + "startDate (%s) > endDate (%s)", startDate, endDate));
}
if (startDate != null) {
filterStartDate = max(startDate, lastFileInstant);
} else {
if (mirrorProperties.getNetwork() != MirrorProperties.HederaNetwork.DEMO && lastFileInstant == null) {
filterStartDate = STARTUP_TIME;
}
}
DateRangeFilter filter = new DateRangeFilter(filterStartDate, endDate);
log.info("{}: parser will parse items in the range [{}, {}]", downloaderProperties.getStreamType(), filter.getStartAsInstant(), filter.getEndAsInstant());
return filter;
}
use of com.hedera.mirror.common.domain.StreamType in project hedera-mirror-node by hashgraph.
the class MirrorDateRangePropertiesProcessorTest method startDateSetAndDatabaseEmpty.
@Test
void startDateSetAndDatabaseEmpty() {
var startDate = STARTUP_TIME.plusSeconds(10L);
mirrorProperties.setStartDate(startDate);
DateRangeFilter expectedFilter = new DateRangeFilter(mirrorProperties.getStartDate(), null);
Instant expectedDate = mirrorProperties.getStartDate();
for (var downloaderProperties : downloaderPropertiesList) {
StreamType streamType = downloaderProperties.getStreamType();
assertThat(mirrorDateRangePropertiesProcessor.getLastStreamFile(streamType)).isEqualTo(streamFile(streamType, expectedDate));
assertThat(mirrorDateRangePropertiesProcessor.getDateRangeFilter(streamType)).isEqualTo(expectedFilter);
}
assertThat(mirrorProperties.getVerifyHashAfter()).isEqualTo(startDate);
}
Aggregations