Search in sources :

Example 1 with StreamType

use of com.hedera.mirror.common.domain.StreamType in project hedera-mirror-node by hashgraph.

the class StreamFilename method extractTypeInfo.

private static TypeInfo extractTypeInfo(String filename) {
    List<String> parts = FILENAME_SPLITTER.splitToList(filename);
    if (parts.size() < 2) {
        throw new InvalidStreamFileException("Failed to determine StreamType for filename: " + filename);
    }
    String last = parts.get(parts.size() - 1);
    String secondLast = parts.get(parts.size() - 2);
    for (StreamType type : StreamType.values()) {
        String suffix = type.getSuffix();
        if (!StringUtils.isEmpty(suffix) && !filename.contains(suffix)) {
            continue;
        }
        Map<String, StreamType.Extension> extensions = STREAM_TYPE_EXTENSION_MAP.get(type);
        if (extensions.containsKey(last)) {
            // if last matches extension, the file is not compressed
            FileType fileType = last.endsWith(StreamType.SIGNATURE_SUFFIX) ? SIGNATURE : DATA;
            return TypeInfo.of(null, extensions.get(last), fileType, type);
        }
        if (extensions.containsKey(secondLast)) {
            // otherwise if secondLast matches extension, last is the compression extension
            FileType fileType = secondLast.endsWith(StreamType.SIGNATURE_SUFFIX) ? SIGNATURE : DATA;
            return TypeInfo.of(last, extensions.get(secondLast), fileType, type);
        }
    }
    throw new InvalidStreamFileException("Failed to determine StreamType for filename: " + filename);
}
Also used : StreamType(com.hedera.mirror.common.domain.StreamType) InvalidStreamFileException(com.hedera.mirror.importer.exception.InvalidStreamFileException)

Example 2 with StreamType

use of com.hedera.mirror.common.domain.StreamType in project hedera-mirror-node by hashgraph.

the class NodeSignatureVerifier method statusMap.

private Map<String, Collection<String>> statusMap(Collection<FileStreamSignature> signatures, Map<String, PublicKey> nodeAccountIDPubKeyMap) {
    Map<String, Collection<String>> statusMap = signatures.stream().collect(Collectors.groupingBy(fss -> fss.getStatus().toString(), Collectors.mapping(fss -> fss.getNodeAccountIdString(), Collectors.toCollection(TreeSet::new))));
    Set<String> seenNodes = new HashSet<>();
    signatures.forEach(signature -> seenNodes.add(signature.getNodeAccountId().entityIdToString()));
    Set<String> missingNodes = new TreeSet<>(Sets.difference(nodeAccountIDPubKeyMap.keySet().stream().collect(Collectors.toSet()), seenNodes));
    statusMap.put(SignatureStatus.NOT_FOUND.toString(), missingNodes);
    String streamType = signatures.stream().map(FileStreamSignature::getStreamType).map(StreamType::toString).findFirst().orElse("UNKNOWN");
    for (Map.Entry<String, Collection<String>> entry : statusMap.entrySet()) {
        entry.getValue().forEach(nodeAccountId -> {
            Counter counter = nodeSignatureStatusMetricMap.computeIfAbsent(nodeAccountId, n -> newStatusMetric(nodeAccountId, streamType, entry.getKey()));
            counter.increment();
        });
    }
    // remove CONSENSUS_REACHED for logging purposes
    statusMap.remove(SignatureStatus.CONSENSUS_REACHED.toString());
    return statusMap;
}
Also used : Counter(io.micrometer.core.instrument.Counter) EntityId(com.hedera.mirror.common.domain.entity.EntityId) AddressBook(com.hedera.mirror.common.domain.addressbook.AddressBook) Collection(java.util.Collection) StreamType(com.hedera.mirror.common.domain.StreamType) Signature(java.security.Signature) ConcurrentHashMap(java.util.concurrent.ConcurrentHashMap) Set(java.util.Set) PublicKey(java.security.PublicKey) Multimap(com.google.common.collect.Multimap) Collectors(java.util.stream.Collectors) EntityType(com.hedera.mirror.common.domain.entity.EntityType) SignatureVerificationException(com.hedera.mirror.importer.exception.SignatureVerificationException) Sets(com.google.common.collect.Sets) TreeSet(java.util.TreeSet) HashSet(java.util.HashSet) FileStreamSignature(com.hedera.mirror.importer.domain.FileStreamSignature) AddressBookService(com.hedera.mirror.importer.addressbook.AddressBookService) HashMultimap(com.google.common.collect.HashMultimap) MeterRegistry(io.micrometer.core.instrument.MeterRegistry) Map(java.util.Map) Log4j2(lombok.extern.log4j.Log4j2) SignatureStatus(com.hedera.mirror.importer.domain.FileStreamSignature.SignatureStatus) Named(javax.inject.Named) StreamType(com.hedera.mirror.common.domain.StreamType) Counter(io.micrometer.core.instrument.Counter) TreeSet(java.util.TreeSet) Collection(java.util.Collection) ConcurrentHashMap(java.util.concurrent.ConcurrentHashMap) Map(java.util.Map) HashSet(java.util.HashSet)

Example 3 with StreamType

use of com.hedera.mirror.common.domain.StreamType in project hedera-mirror-node by hashgraph.

the class MirrorDateRangePropertiesProcessor method getLastStreamFile.

/**
 * Gets the latest stream file for downloader based on startDate in MirrorProperties, the startDateAdjustment and
 * last valid downloaded stream file.
 *
 * @param streamType What type of stream to retrieve
 * @return The latest stream file from the database or a dummy stream file if it calculated a different effective
 * start date
 */
public <T extends StreamFile> Optional<T> getLastStreamFile(StreamType streamType) {
    Instant startDate = mirrorProperties.getStartDate();
    Optional<T> streamFile = findLatest(streamType);
    Instant lastFileInstant = streamFile.map(StreamFile::getConsensusStart).map(nanos -> Instant.ofEpochSecond(0, nanos)).orElse(null);
    Instant effectiveStartDate = STARTUP_TIME;
    boolean hasStreamFile = lastFileInstant != null;
    if (startDate != null) {
        effectiveStartDate = max(startDate, hasStreamFile ? lastFileInstant : Instant.EPOCH);
    } else if (hasStreamFile) {
        effectiveStartDate = lastFileInstant;
    } else if (mirrorProperties.getNetwork() == MirrorProperties.HederaNetwork.DEMO) {
        // Demo network contains only data in the past, so don't default to now
        effectiveStartDate = Instant.EPOCH;
    }
    Instant endDate = mirrorProperties.getEndDate();
    if (startDate != null && startDate.compareTo(endDate) > 0) {
        throw new InvalidConfigurationException(String.format("Date range constraint violation: " + "startDate (%s) > endDate (%s)", startDate, endDate));
    }
    if (effectiveStartDate.compareTo(endDate) > 0) {
        throw new InvalidConfigurationException(String.format("Date range constraint violation for %s downloader: effective startDate (%s) > endDate (%s)", streamType, effectiveStartDate, endDate));
    }
    if (!effectiveStartDate.equals(lastFileInstant)) {
        Instant verifyHashAfter = mirrorProperties.getVerifyHashAfter();
        if (verifyHashAfter == null || verifyHashAfter.isBefore(effectiveStartDate)) {
            mirrorProperties.setVerifyHashAfter(effectiveStartDate);
            log.debug("Set verifyHashAfter to {}", effectiveStartDate);
        }
        String filename = StreamFilename.getFilename(streamType, DATA, effectiveStartDate);
        T effectiveStreamFile = (T) ReflectUtils.newInstance(streamType.getStreamFileClass());
        effectiveStreamFile.setConsensusStart(DomainUtils.convertToNanosMax(effectiveStartDate));
        effectiveStreamFile.setName(filename);
        effectiveStreamFile.setIndex(streamFile.map(StreamFile::getIndex).orElse(null));
        streamFile = Optional.of(effectiveStreamFile);
    }
    log.info("{}: downloader will download files in time range ({}, {}]", streamType, effectiveStartDate, mirrorProperties.getEndDate());
    return streamFile;
}
Also used : DownloaderProperties(com.hedera.mirror.importer.downloader.DownloaderProperties) InvalidConfigurationException(com.hedera.mirror.importer.exception.InvalidConfigurationException) RequiredArgsConstructor(lombok.RequiredArgsConstructor) Value(lombok.Value) StreamFilename(com.hedera.mirror.importer.domain.StreamFilename) Duration(java.time.Duration) Map(java.util.Map) Named(javax.inject.Named) Utility(com.hedera.mirror.importer.util.Utility) DomainUtils(com.hedera.mirror.common.util.DomainUtils) ObjectUtils.max(org.apache.commons.lang3.ObjectUtils.max) EventFileRepository(com.hedera.mirror.importer.repository.EventFileRepository) ReflectUtils(org.springframework.cglib.core.ReflectUtils) DATA(com.hedera.mirror.importer.domain.StreamFilename.FileType.DATA) StreamType(com.hedera.mirror.common.domain.StreamType) ConcurrentHashMap(java.util.concurrent.ConcurrentHashMap) StreamFile(com.hedera.mirror.common.domain.StreamFile) Instant(java.time.Instant) AccountBalanceFileRepository(com.hedera.mirror.importer.repository.AccountBalanceFileRepository) List(java.util.List) MirrorProperties(com.hedera.mirror.importer.MirrorProperties) RecordFileRepository(com.hedera.mirror.importer.repository.RecordFileRepository) Log4j2(lombok.extern.log4j.Log4j2) Optional(java.util.Optional) StreamFileRepository(com.hedera.mirror.importer.repository.StreamFileRepository) Instant(java.time.Instant) StreamFile(com.hedera.mirror.common.domain.StreamFile) InvalidConfigurationException(com.hedera.mirror.importer.exception.InvalidConfigurationException)

Example 4 with StreamType

use of com.hedera.mirror.common.domain.StreamType in project hedera-mirror-node by hashgraph.

the class MirrorDateRangePropertiesProcessor method newDateRangeFilter.

private DateRangeFilter newDateRangeFilter(StreamType streamType) {
    DownloaderProperties downloaderProperties = getDownloaderProperties(streamType);
    if (!downloaderProperties.isEnabled()) {
        return DateRangeFilter.empty();
    }
    Instant startDate = mirrorProperties.getStartDate();
    Instant endDate = mirrorProperties.getEndDate();
    Instant lastFileInstant = findLatest(streamType).map(StreamFile::getConsensusStart).map(nanos -> Instant.ofEpochSecond(0, nanos)).orElse(null);
    Instant filterStartDate = lastFileInstant;
    if (startDate != null && startDate.compareTo(endDate) > 0) {
        throw new InvalidConfigurationException(String.format("Date range constraint violation: " + "startDate (%s) > endDate (%s)", startDate, endDate));
    }
    if (startDate != null) {
        filterStartDate = max(startDate, lastFileInstant);
    } else {
        if (mirrorProperties.getNetwork() != MirrorProperties.HederaNetwork.DEMO && lastFileInstant == null) {
            filterStartDate = STARTUP_TIME;
        }
    }
    DateRangeFilter filter = new DateRangeFilter(filterStartDate, endDate);
    log.info("{}: parser will parse items in the range [{}, {}]", downloaderProperties.getStreamType(), filter.getStartAsInstant(), filter.getEndAsInstant());
    return filter;
}
Also used : DownloaderProperties(com.hedera.mirror.importer.downloader.DownloaderProperties) InvalidConfigurationException(com.hedera.mirror.importer.exception.InvalidConfigurationException) RequiredArgsConstructor(lombok.RequiredArgsConstructor) Value(lombok.Value) StreamFilename(com.hedera.mirror.importer.domain.StreamFilename) Duration(java.time.Duration) Map(java.util.Map) Named(javax.inject.Named) Utility(com.hedera.mirror.importer.util.Utility) DomainUtils(com.hedera.mirror.common.util.DomainUtils) ObjectUtils.max(org.apache.commons.lang3.ObjectUtils.max) EventFileRepository(com.hedera.mirror.importer.repository.EventFileRepository) ReflectUtils(org.springframework.cglib.core.ReflectUtils) DATA(com.hedera.mirror.importer.domain.StreamFilename.FileType.DATA) StreamType(com.hedera.mirror.common.domain.StreamType) ConcurrentHashMap(java.util.concurrent.ConcurrentHashMap) StreamFile(com.hedera.mirror.common.domain.StreamFile) Instant(java.time.Instant) AccountBalanceFileRepository(com.hedera.mirror.importer.repository.AccountBalanceFileRepository) List(java.util.List) MirrorProperties(com.hedera.mirror.importer.MirrorProperties) RecordFileRepository(com.hedera.mirror.importer.repository.RecordFileRepository) Log4j2(lombok.extern.log4j.Log4j2) Optional(java.util.Optional) StreamFileRepository(com.hedera.mirror.importer.repository.StreamFileRepository) DownloaderProperties(com.hedera.mirror.importer.downloader.DownloaderProperties) Instant(java.time.Instant) StreamFile(com.hedera.mirror.common.domain.StreamFile) InvalidConfigurationException(com.hedera.mirror.importer.exception.InvalidConfigurationException)

Example 5 with StreamType

use of com.hedera.mirror.common.domain.StreamType in project hedera-mirror-node by hashgraph.

the class MirrorDateRangePropertiesProcessorTest method startDateSetAndDatabaseEmpty.

@Test
void startDateSetAndDatabaseEmpty() {
    var startDate = STARTUP_TIME.plusSeconds(10L);
    mirrorProperties.setStartDate(startDate);
    DateRangeFilter expectedFilter = new DateRangeFilter(mirrorProperties.getStartDate(), null);
    Instant expectedDate = mirrorProperties.getStartDate();
    for (var downloaderProperties : downloaderPropertiesList) {
        StreamType streamType = downloaderProperties.getStreamType();
        assertThat(mirrorDateRangePropertiesProcessor.getLastStreamFile(streamType)).isEqualTo(streamFile(streamType, expectedDate));
        assertThat(mirrorDateRangePropertiesProcessor.getDateRangeFilter(streamType)).isEqualTo(expectedFilter);
    }
    assertThat(mirrorProperties.getVerifyHashAfter()).isEqualTo(startDate);
}
Also used : StreamType(com.hedera.mirror.common.domain.StreamType) Instant(java.time.Instant) DateRangeFilter(com.hedera.mirror.importer.config.MirrorDateRangePropertiesProcessor.DateRangeFilter) Test(org.junit.jupiter.api.Test) ParameterizedTest(org.junit.jupiter.params.ParameterizedTest)

Aggregations

StreamType (com.hedera.mirror.common.domain.StreamType)12 Instant (java.time.Instant)9 ParameterizedTest (org.junit.jupiter.params.ParameterizedTest)8 DateRangeFilter (com.hedera.mirror.importer.config.MirrorDateRangePropertiesProcessor.DateRangeFilter)7 Test (org.junit.jupiter.api.Test)5 StreamFile (com.hedera.mirror.common.domain.StreamFile)3 Map (java.util.Map)3 ConcurrentHashMap (java.util.concurrent.ConcurrentHashMap)3 Named (javax.inject.Named)3 Log4j2 (lombok.extern.log4j.Log4j2)3 DomainUtils (com.hedera.mirror.common.util.DomainUtils)2 MirrorProperties (com.hedera.mirror.importer.MirrorProperties)2 StreamFilename (com.hedera.mirror.importer.domain.StreamFilename)2 DATA (com.hedera.mirror.importer.domain.StreamFilename.FileType.DATA)2 DownloaderProperties (com.hedera.mirror.importer.downloader.DownloaderProperties)2 InvalidConfigurationException (com.hedera.mirror.importer.exception.InvalidConfigurationException)2 AccountBalanceFileRepository (com.hedera.mirror.importer.repository.AccountBalanceFileRepository)2 EventFileRepository (com.hedera.mirror.importer.repository.EventFileRepository)2 RecordFileRepository (com.hedera.mirror.importer.repository.RecordFileRepository)2 StreamFileRepository (com.hedera.mirror.importer.repository.StreamFileRepository)2