use of com.hedera.mirror.common.domain.StreamFile in project hedera-mirror-node by hashgraph.
the class MirrorDateRangePropertiesProcessor method getLastStreamFile.
/**
* Gets the latest stream file for downloader based on startDate in MirrorProperties, the startDateAdjustment and
* last valid downloaded stream file.
*
* @param streamType What type of stream to retrieve
* @return The latest stream file from the database or a dummy stream file if it calculated a different effective
* start date
*/
public <T extends StreamFile> Optional<T> getLastStreamFile(StreamType streamType) {
Instant startDate = mirrorProperties.getStartDate();
Optional<T> streamFile = findLatest(streamType);
Instant lastFileInstant = streamFile.map(StreamFile::getConsensusStart).map(nanos -> Instant.ofEpochSecond(0, nanos)).orElse(null);
Instant effectiveStartDate = STARTUP_TIME;
boolean hasStreamFile = lastFileInstant != null;
if (startDate != null) {
effectiveStartDate = max(startDate, hasStreamFile ? lastFileInstant : Instant.EPOCH);
} else if (hasStreamFile) {
effectiveStartDate = lastFileInstant;
} else if (mirrorProperties.getNetwork() == MirrorProperties.HederaNetwork.DEMO) {
// Demo network contains only data in the past, so don't default to now
effectiveStartDate = Instant.EPOCH;
}
Instant endDate = mirrorProperties.getEndDate();
if (startDate != null && startDate.compareTo(endDate) > 0) {
throw new InvalidConfigurationException(String.format("Date range constraint violation: " + "startDate (%s) > endDate (%s)", startDate, endDate));
}
if (effectiveStartDate.compareTo(endDate) > 0) {
throw new InvalidConfigurationException(String.format("Date range constraint violation for %s downloader: effective startDate (%s) > endDate (%s)", streamType, effectiveStartDate, endDate));
}
if (!effectiveStartDate.equals(lastFileInstant)) {
Instant verifyHashAfter = mirrorProperties.getVerifyHashAfter();
if (verifyHashAfter == null || verifyHashAfter.isBefore(effectiveStartDate)) {
mirrorProperties.setVerifyHashAfter(effectiveStartDate);
log.debug("Set verifyHashAfter to {}", effectiveStartDate);
}
String filename = StreamFilename.getFilename(streamType, DATA, effectiveStartDate);
T effectiveStreamFile = (T) ReflectUtils.newInstance(streamType.getStreamFileClass());
effectiveStreamFile.setConsensusStart(DomainUtils.convertToNanosMax(effectiveStartDate));
effectiveStreamFile.setName(filename);
effectiveStreamFile.setIndex(streamFile.map(StreamFile::getIndex).orElse(null));
streamFile = Optional.of(effectiveStreamFile);
}
log.info("{}: downloader will download files in time range ({}, {}]", streamType, effectiveStartDate, mirrorProperties.getEndDate());
return streamFile;
}
use of com.hedera.mirror.common.domain.StreamFile in project hedera-mirror-node by hashgraph.
the class MirrorDateRangePropertiesProcessor method newDateRangeFilter.
private DateRangeFilter newDateRangeFilter(StreamType streamType) {
DownloaderProperties downloaderProperties = getDownloaderProperties(streamType);
if (!downloaderProperties.isEnabled()) {
return DateRangeFilter.empty();
}
Instant startDate = mirrorProperties.getStartDate();
Instant endDate = mirrorProperties.getEndDate();
Instant lastFileInstant = findLatest(streamType).map(StreamFile::getConsensusStart).map(nanos -> Instant.ofEpochSecond(0, nanos)).orElse(null);
Instant filterStartDate = lastFileInstant;
if (startDate != null && startDate.compareTo(endDate) > 0) {
throw new InvalidConfigurationException(String.format("Date range constraint violation: " + "startDate (%s) > endDate (%s)", startDate, endDate));
}
if (startDate != null) {
filterStartDate = max(startDate, lastFileInstant);
} else {
if (mirrorProperties.getNetwork() != MirrorProperties.HederaNetwork.DEMO && lastFileInstant == null) {
filterStartDate = STARTUP_TIME;
}
}
DateRangeFilter filter = new DateRangeFilter(filterStartDate, endDate);
log.info("{}: parser will parse items in the range [{}, {}]", downloaderProperties.getStreamType(), filter.getStartAsInstant(), filter.getEndAsInstant());
return filter;
}
use of com.hedera.mirror.common.domain.StreamFile in project hedera-mirror-node by hashgraph.
the class AbstractDownloaderTest method verifyStreamFiles.
protected void verifyStreamFiles(List<String> files) {
ArgumentCaptor<StreamFile> captor = ArgumentCaptor.forClass(StreamFile.class);
AtomicLong index = new AtomicLong(firstIndex);
verify(streamFileNotifier, times(files.size())).verified(captor.capture());
assertThat(captor.getAllValues()).allMatch(s -> files.contains(s.getName())).allMatch(s -> s.getIndex() == null || s.getIndex() == index.getAndIncrement()).allMatch(s -> downloaderProperties.isPersistBytes() ^ (s.getBytes() == null));
}
use of com.hedera.mirror.common.domain.StreamFile in project hedera-mirror-node by hashgraph.
the class AbstractLinkedStreamDownloaderTest method verifyHashChain.
@ParameterizedTest(name = "verifyHashChain {5}")
@CsvSource({ // @formatter:off
"'', '', 1970-01-01T00:00:00Z, 2000-01-01T10:00:00Z, true, passes if both hashes are empty", // starting stream in middle
"xx, '', 1970-01-01T00:00:00Z, 2000-01-01T10:00:00Z, true, passes if hash mismatch and expected hash is empty", // bad db state
"'', xx, 1970-01-01T00:00:00Z, 2000-01-01T10:00:00Z, false, fails if hash mismatch and actual hash is empty", "xx, yy, 1970-01-01T00:00:00Z, 2000-01-01T10:00:00Z, false, fails if hash mismatch and hashes are non-empty", "xx, yy, 2000-01-01T10:00:00.000001Z, 2000-01-01T10:00:00Z, true, passes if hash mismatch but verifyHashAfter is after filename", "xx, yy, 2000-01-01T10:00:00.000001Z, 2000-01-01T10:00:00Z, true, passes if hash mismatch but verifyHashAfter is same as filename", "xx, yy, 2000-01-01T09:59:59.999999Z, 2000-01-01T10:00:00Z, false, fails if hash mismatch and verifyHashAfter is before filename", "xx, xx, 1970-01-01T00:00:00Z, 2000-01-01T10:00:00Z, true, passes if hashes are equal" // @formatter:on
})
void verifyHashChain(String actualPrevFileHash, String expectedPrevFileHash, Instant verifyHashAfter, Instant fileInstant, Boolean expectedResult, String testName) {
downloaderProperties.getMirrorProperties().setVerifyHashAfter(verifyHashAfter);
StreamFile streamFile = (StreamFile) ReflectUtils.newInstance(streamType.getStreamFileClass());
streamFile.setConsensusStart(DomainUtils.convertToNanosMax(fileInstant));
streamFile.setName(StreamFilename.getFilename(streamType, StreamFilename.FileType.DATA, fileInstant));
streamFile.setPreviousHash(actualPrevFileHash);
assertThat(downloader.verifyHashChain(streamFile, expectedPrevFileHash)).as(testName).isEqualTo(expectedResult);
}
use of com.hedera.mirror.common.domain.StreamFile in project hedera-mirror-node by hashgraph.
the class AbstractStreamFileParserTest method failureShouldRollback.
@Test
void failureShouldRollback() {
// given
StreamFile streamFile = getStreamFile();
mockDbFailure();
// when
Assertions.assertThrows(ParserException.class, () -> {
parser.parse(streamFile);
});
// then
assertParsed(streamFile, false, true);
assertPostParseStreamFile(streamFile, false);
}
Aggregations