Examples with StorageUsage org.apache.nifi.diagnostics.StorageUsage used on opensource projects

Example 1 with StorageUsage

use of org.apache.nifi.diagnostics.StorageUsage in project nifi by apache.

the class DtoFactory method createSystemDiagnosticsDto.

/**
 * Creates a SystemDiagnosticsDTO for the specified SystemDiagnostics.
 *
 * @param sysDiagnostics diags
 * @return dto
 */
public SystemDiagnosticsDTO createSystemDiagnosticsDto(final SystemDiagnostics sysDiagnostics) {
    final SystemDiagnosticsDTO dto = new SystemDiagnosticsDTO();
    final SystemDiagnosticsSnapshotDTO snapshot = new SystemDiagnosticsSnapshotDTO();
    dto.setAggregateSnapshot(snapshot);
    snapshot.setStatsLastRefreshed(new Date(sysDiagnostics.getCreationTimestamp()));
    // processors
    snapshot.setAvailableProcessors(sysDiagnostics.getAvailableProcessors());
    snapshot.setProcessorLoadAverage(sysDiagnostics.getProcessorLoadAverage());
    // threads
    snapshot.setDaemonThreads(sysDiagnostics.getDaemonThreads());
    snapshot.setTotalThreads(sysDiagnostics.getTotalThreads());
    // heap
    snapshot.setMaxHeap(FormatUtils.formatDataSize(sysDiagnostics.getMaxHeap()));
    snapshot.setMaxHeapBytes(sysDiagnostics.getMaxHeap());
    snapshot.setTotalHeap(FormatUtils.formatDataSize(sysDiagnostics.getTotalHeap()));
    snapshot.setTotalHeapBytes(sysDiagnostics.getTotalHeap());
    snapshot.setUsedHeap(FormatUtils.formatDataSize(sysDiagnostics.getUsedHeap()));
    snapshot.setUsedHeapBytes(sysDiagnostics.getUsedHeap());
    snapshot.setFreeHeap(FormatUtils.formatDataSize(sysDiagnostics.getFreeHeap()));
    snapshot.setFreeHeapBytes(sysDiagnostics.getFreeHeap());
    if (sysDiagnostics.getHeapUtilization() != -1) {
        snapshot.setHeapUtilization(FormatUtils.formatUtilization(sysDiagnostics.getHeapUtilization()));
    }
    // non heap
    snapshot.setMaxNonHeap(FormatUtils.formatDataSize(sysDiagnostics.getMaxNonHeap()));
    snapshot.setMaxNonHeapBytes(sysDiagnostics.getMaxNonHeap());
    snapshot.setTotalNonHeap(FormatUtils.formatDataSize(sysDiagnostics.getTotalNonHeap()));
    snapshot.setTotalNonHeapBytes(sysDiagnostics.getTotalNonHeap());
    snapshot.setUsedNonHeap(FormatUtils.formatDataSize(sysDiagnostics.getUsedNonHeap()));
    snapshot.setUsedNonHeapBytes(sysDiagnostics.getUsedNonHeap());
    snapshot.setFreeNonHeap(FormatUtils.formatDataSize(sysDiagnostics.getFreeNonHeap()));
    snapshot.setFreeNonHeapBytes(sysDiagnostics.getFreeNonHeap());
    if (sysDiagnostics.getNonHeapUtilization() != -1) {
        snapshot.setNonHeapUtilization(FormatUtils.formatUtilization(sysDiagnostics.getNonHeapUtilization()));
    }
    // flow file disk usage
    final SystemDiagnosticsSnapshotDTO.StorageUsageDTO flowFileRepositoryStorageUsageDto = createStorageUsageDTO(null, sysDiagnostics.getFlowFileRepositoryStorageUsage());
    snapshot.setFlowFileRepositoryStorageUsage(flowFileRepositoryStorageUsageDto);
    // content disk usage
    final Set<SystemDiagnosticsSnapshotDTO.StorageUsageDTO> contentRepositoryStorageUsageDtos = new LinkedHashSet<>();
    snapshot.setContentRepositoryStorageUsage(contentRepositoryStorageUsageDtos);
    for (final Map.Entry<String, StorageUsage> entry : sysDiagnostics.getContentRepositoryStorageUsage().entrySet()) {
        contentRepositoryStorageUsageDtos.add(createStorageUsageDTO(entry.getKey(), entry.getValue()));
    }
    // provenance disk usage
    final Set<SystemDiagnosticsSnapshotDTO.StorageUsageDTO> provenanceRepositoryStorageUsageDtos = new LinkedHashSet<>();
    snapshot.setProvenanceRepositoryStorageUsage(provenanceRepositoryStorageUsageDtos);
    for (final Map.Entry<String, StorageUsage> entry : sysDiagnostics.getProvenanceRepositoryStorageUsage().entrySet()) {
        provenanceRepositoryStorageUsageDtos.add(createStorageUsageDTO(entry.getKey(), entry.getValue()));
    }
    // garbage collection
    final Set<SystemDiagnosticsSnapshotDTO.GarbageCollectionDTO> garbageCollectionDtos = new LinkedHashSet<>();
    snapshot.setGarbageCollection(garbageCollectionDtos);
    for (final Map.Entry<String, GarbageCollection> entry : sysDiagnostics.getGarbageCollection().entrySet()) {
        garbageCollectionDtos.add(createGarbageCollectionDTO(entry.getKey(), entry.getValue()));
    }
    // version info
    final SystemDiagnosticsSnapshotDTO.VersionInfoDTO versionInfoDto = createVersionInfoDTO();
    snapshot.setVersionInfo(versionInfoDto);
    // uptime
    snapshot.setUptime(FormatUtils.formatHoursMinutesSeconds(sysDiagnostics.getUptime(), TimeUnit.MILLISECONDS));
    return dto;
}

Example 2 with StorageUsage

use of org.apache.nifi.diagnostics.StorageUsage in project nifi by apache.

the class DtoFactory method createJvmDiagnosticsSnapshotDto.

private JVMDiagnosticsSnapshotDTO createJvmDiagnosticsSnapshotDto(final FlowController flowController) {
    final JVMDiagnosticsSnapshotDTO dto = new JVMDiagnosticsSnapshotDTO();
    final JVMControllerDiagnosticsSnapshotDTO controllerDiagnosticsDto = new JVMControllerDiagnosticsSnapshotDTO();
    final JVMFlowDiagnosticsSnapshotDTO flowDiagnosticsDto = new JVMFlowDiagnosticsSnapshotDTO();
    final JVMSystemDiagnosticsSnapshotDTO systemDiagnosticsDto = new JVMSystemDiagnosticsSnapshotDTO();
    dto.setControllerDiagnostics(controllerDiagnosticsDto);
    dto.setFlowDiagnosticsDto(flowDiagnosticsDto);
    dto.setSystemDiagnosticsDto(systemDiagnosticsDto);
    final SystemDiagnostics systemDiagnostics = flowController.getSystemDiagnostics();
    // flow-related information
    final Set<BundleDTO> bundlesLoaded = ExtensionManager.getAllBundles().stream().map(bundle -> bundle.getBundleDetails().getCoordinate()).sorted((a, b) -> a.getCoordinate().compareTo(b.getCoordinate())).map(this::createBundleDto).collect(Collectors.toCollection(LinkedHashSet::new));
    flowDiagnosticsDto.setActiveEventDrivenThreads(flowController.getActiveEventDrivenThreadCount());
    flowDiagnosticsDto.setActiveTimerDrivenThreads(flowController.getActiveTimerDrivenThreadCount());
    flowDiagnosticsDto.setBundlesLoaded(bundlesLoaded);
    flowDiagnosticsDto.setTimeZone(System.getProperty("user.timezone"));
    flowDiagnosticsDto.setUptime(FormatUtils.formatHoursMinutesSeconds(systemDiagnostics.getUptime(), TimeUnit.MILLISECONDS));
    // controller-related information
    controllerDiagnosticsDto.setClusterCoordinator(flowController.isClusterCoordinator());
    controllerDiagnosticsDto.setPrimaryNode(flowController.isPrimary());
    controllerDiagnosticsDto.setMaxEventDrivenThreads(flowController.getMaxEventDrivenThreadCount());
    controllerDiagnosticsDto.setMaxTimerDrivenThreads(flowController.getMaxTimerDrivenThreadCount());
    // system-related information
    systemDiagnosticsDto.setMaxOpenFileDescriptors(systemDiagnostics.getMaxOpenFileHandles());
    systemDiagnosticsDto.setOpenFileDescriptors(systemDiagnostics.getOpenFileHandles());
    systemDiagnosticsDto.setPhysicalMemoryBytes(systemDiagnostics.getTotalPhysicalMemory());
    systemDiagnosticsDto.setPhysicalMemory(FormatUtils.formatDataSize(systemDiagnostics.getTotalPhysicalMemory()));
    final NumberFormat percentageFormat = NumberFormat.getPercentInstance();
    percentageFormat.setMaximumFractionDigits(2);
    final Set<RepositoryUsageDTO> contentRepoUsage = new HashSet<>();
    for (final Map.Entry<String, StorageUsage> entry : systemDiagnostics.getContentRepositoryStorageUsage().entrySet()) {
        final String repoName = entry.getKey();
        final StorageUsage usage = entry.getValue();
        final RepositoryUsageDTO usageDto = new RepositoryUsageDTO();
        usageDto.setName(repoName);
        usageDto.setFileStoreHash(DigestUtils.sha256Hex(flowController.getContentRepoFileStoreName(repoName)));
        usageDto.setFreeSpace(FormatUtils.formatDataSize(usage.getFreeSpace()));
        usageDto.setFreeSpaceBytes(usage.getFreeSpace());
        usageDto.setTotalSpace(FormatUtils.formatDataSize(usage.getTotalSpace()));
        usageDto.setTotalSpaceBytes(usage.getTotalSpace());
        final double usedPercentage = (usage.getTotalSpace() - usage.getFreeSpace()) / (double) usage.getTotalSpace();
        final String utilization = percentageFormat.format(usedPercentage);
        usageDto.setUtilization(utilization);
        contentRepoUsage.add(usageDto);
    }
    final Set<RepositoryUsageDTO> provRepoUsage = new HashSet<>();
    for (final Map.Entry<String, StorageUsage> entry : systemDiagnostics.getProvenanceRepositoryStorageUsage().entrySet()) {
        final String repoName = entry.getKey();
        final StorageUsage usage = entry.getValue();
        final RepositoryUsageDTO usageDto = new RepositoryUsageDTO();
        usageDto.setName(repoName);
        usageDto.setFileStoreHash(DigestUtils.sha256Hex(flowController.getProvenanceRepoFileStoreName(repoName)));
        usageDto.setFreeSpace(FormatUtils.formatDataSize(usage.getFreeSpace()));
        usageDto.setFreeSpaceBytes(usage.getFreeSpace());
        usageDto.setTotalSpace(FormatUtils.formatDataSize(usage.getTotalSpace()));
        usageDto.setTotalSpaceBytes(usage.getTotalSpace());
        final double usedPercentage = (usage.getTotalSpace() - usage.getFreeSpace()) / (double) usage.getTotalSpace();
        final String utilization = percentageFormat.format(usedPercentage);
        usageDto.setUtilization(utilization);
        provRepoUsage.add(usageDto);
    }
    final RepositoryUsageDTO flowFileRepoUsage = new RepositoryUsageDTO();
    for (final Map.Entry<String, StorageUsage> entry : systemDiagnostics.getProvenanceRepositoryStorageUsage().entrySet()) {
        final String repoName = entry.getKey();
        final StorageUsage usage = entry.getValue();
        flowFileRepoUsage.setName(repoName);
        flowFileRepoUsage.setFileStoreHash(DigestUtils.sha256Hex(flowController.getFlowRepoFileStoreName()));
        flowFileRepoUsage.setFreeSpace(FormatUtils.formatDataSize(usage.getFreeSpace()));
        flowFileRepoUsage.setFreeSpaceBytes(usage.getFreeSpace());
        flowFileRepoUsage.setTotalSpace(FormatUtils.formatDataSize(usage.getTotalSpace()));
        flowFileRepoUsage.setTotalSpaceBytes(usage.getTotalSpace());
        final double usedPercentage = (usage.getTotalSpace() - usage.getFreeSpace()) / (double) usage.getTotalSpace();
        final String utilization = percentageFormat.format(usedPercentage);
        flowFileRepoUsage.setUtilization(utilization);
    }
    systemDiagnosticsDto.setContentRepositoryStorageUsage(contentRepoUsage);
    systemDiagnosticsDto.setCpuCores(systemDiagnostics.getAvailableProcessors());
    systemDiagnosticsDto.setCpuLoadAverage(systemDiagnostics.getProcessorLoadAverage());
    systemDiagnosticsDto.setFlowFileRepositoryStorageUsage(flowFileRepoUsage);
    systemDiagnosticsDto.setMaxHeapBytes(systemDiagnostics.getMaxHeap());
    systemDiagnosticsDto.setMaxHeap(FormatUtils.formatDataSize(systemDiagnostics.getMaxHeap()));
    systemDiagnosticsDto.setProvenanceRepositoryStorageUsage(provRepoUsage);
    // Create the Garbage Collection History info
    final GarbageCollectionHistory gcHistory = flowController.getGarbageCollectionHistory();
    final List<GarbageCollectionDiagnosticsDTO> gcDiagnostics = new ArrayList<>();
    for (final String memoryManager : gcHistory.getMemoryManagerNames()) {
        final List<GarbageCollectionStatus> statuses = gcHistory.getGarbageCollectionStatuses(memoryManager);
        final List<GCDiagnosticsSnapshotDTO> gcSnapshots = new ArrayList<>();
        for (final GarbageCollectionStatus status : statuses) {
            final GCDiagnosticsSnapshotDTO snapshotDto = new GCDiagnosticsSnapshotDTO();
            snapshotDto.setTimestamp(status.getTimestamp());
            snapshotDto.setCollectionCount(status.getCollectionCount());
            snapshotDto.setCollectionMillis(status.getCollectionMillis());
            gcSnapshots.add(snapshotDto);
        }
        final GarbageCollectionDiagnosticsDTO gcDto = new GarbageCollectionDiagnosticsDTO();
        gcDto.setMemoryManagerName(memoryManager);
        gcDto.setSnapshots(gcSnapshots);
        gcDiagnostics.add(gcDto);
    }
    systemDiagnosticsDto.setGarbageCollectionDiagnostics(gcDiagnostics);
    return dto;
}

Example 3 with StorageUsage

use of org.apache.nifi.diagnostics.StorageUsage in project nifi-minifi by apache.

the class StatusRequestParser method parseSystemDiagnosticsRequest.

static SystemDiagnosticsStatus parseSystemDiagnosticsRequest(SystemDiagnostics inputSystemDiagnostics, String statusTypes) throws StatusRequestException {
    if (inputSystemDiagnostics == null) {
        throw new StatusRequestException("Unable to get system diagnostics");
    }
    SystemDiagnosticsStatus systemDiagnosticsStatus = new SystemDiagnosticsStatus();
    String[] statusSplits = statusTypes.split(",");
    for (String statusType : statusSplits) {
        switch(statusType.toLowerCase().trim()) {
            case "heap":
                HeapStatus heapStatus = new HeapStatus();
                heapStatus.setTotalHeap(inputSystemDiagnostics.getTotalHeap());
                heapStatus.setMaxHeap(inputSystemDiagnostics.getMaxHeap());
                heapStatus.setFreeHeap(inputSystemDiagnostics.getFreeHeap());
                heapStatus.setUsedHeap(inputSystemDiagnostics.getUsedHeap());
                heapStatus.setHeapUtilization(inputSystemDiagnostics.getHeapUtilization());
                heapStatus.setTotalNonHeap(inputSystemDiagnostics.getTotalNonHeap());
                heapStatus.setMaxNonHeap(inputSystemDiagnostics.getMaxNonHeap());
                heapStatus.setFreeNonHeap(inputSystemDiagnostics.getFreeNonHeap());
                heapStatus.setUsedNonHeap(inputSystemDiagnostics.getUsedNonHeap());
                heapStatus.setNonHeapUtilization(inputSystemDiagnostics.getNonHeapUtilization());
                systemDiagnosticsStatus.setHeapStatus(heapStatus);
                break;
            case "processorstats":
                SystemProcessorStats systemProcessorStats = new SystemProcessorStats();
                systemProcessorStats.setAvailableProcessors(inputSystemDiagnostics.getAvailableProcessors());
                systemProcessorStats.setLoadAverage(inputSystemDiagnostics.getProcessorLoadAverage());
                systemDiagnosticsStatus.setProcessorStatus(systemProcessorStats);
                break;
            case "contentrepositoryusage":
                List<ContentRepositoryUsage> contentRepositoryUsageList = new LinkedList<>();
                Map<String, StorageUsage> contentRepoStorage = inputSystemDiagnostics.getContentRepositoryStorageUsage();
                for (Map.Entry<String, StorageUsage> stringStorageUsageEntry : contentRepoStorage.entrySet()) {
                    ContentRepositoryUsage contentRepositoryUsage = new ContentRepositoryUsage();
                    StorageUsage storageUsage = stringStorageUsageEntry.getValue();
                    contentRepositoryUsage.setName(storageUsage.getIdentifier());
                    contentRepositoryUsage.setFreeSpace(storageUsage.getFreeSpace());
                    contentRepositoryUsage.setTotalSpace(storageUsage.getTotalSpace());
                    contentRepositoryUsage.setDiskUtilization(storageUsage.getDiskUtilization());
                    contentRepositoryUsage.setUsedSpace(storageUsage.getUsedSpace());
                    contentRepositoryUsageList.add(contentRepositoryUsage);
                }
                systemDiagnosticsStatus.setContentRepositoryUsageList(contentRepositoryUsageList);
                break;
            case "flowfilerepositoryusage":
                FlowfileRepositoryUsage flowfileRepositoryUsage = new FlowfileRepositoryUsage();
                StorageUsage flowFileRepoStorage = inputSystemDiagnostics.getFlowFileRepositoryStorageUsage();
                flowfileRepositoryUsage.setFreeSpace(flowFileRepoStorage.getFreeSpace());
                flowfileRepositoryUsage.setTotalSpace(flowFileRepoStorage.getTotalSpace());
                flowfileRepositoryUsage.setDiskUtilization(flowFileRepoStorage.getDiskUtilization());
                flowfileRepositoryUsage.setUsedSpace(flowFileRepoStorage.getUsedSpace());
                systemDiagnosticsStatus.setFlowfileRepositoryUsage(flowfileRepositoryUsage);
                break;
            case "garbagecollection":
                List<GarbageCollectionStatus> garbageCollectionStatusList = new LinkedList<>();
                Map<String, GarbageCollection> garbageCollectionMap = inputSystemDiagnostics.getGarbageCollection();
                for (Map.Entry<String, GarbageCollection> stringGarbageCollectionEntry : garbageCollectionMap.entrySet()) {
                    GarbageCollectionStatus garbageCollectionStatus = new GarbageCollectionStatus();
                    GarbageCollection garbageCollection = stringGarbageCollectionEntry.getValue();
                    garbageCollectionStatus.setName(garbageCollection.getName());
                    garbageCollectionStatus.setCollectionCount(garbageCollection.getCollectionCount());
                    garbageCollectionStatus.setCollectionTime(garbageCollection.getCollectionTime());
                    garbageCollectionStatusList.add(garbageCollectionStatus);
                }
                systemDiagnosticsStatus.setGarbageCollectionStatusList(garbageCollectionStatusList);
                break;
        }
    }
    return systemDiagnosticsStatus;
}

Example 4 with StorageUsage

use of org.apache.nifi.diagnostics.StorageUsage in project nifi-minifi by apache.

the class StatusConfigReporterTest method addContentRepoToSystemDiagnostics.

private void addContentRepoToSystemDiagnostics(SystemDiagnostics systemDiagnostics) {
    Map<String, StorageUsage> stringStorageUsageMap = new HashMap<>();
    StorageUsage repoUsage1 = new StorageUsage();
    repoUsage1.setFreeSpace(30);
    repoUsage1.setTotalSpace(100);
    repoUsage1.setIdentifier("Content repo1");
    stringStorageUsageMap.put(repoUsage1.getIdentifier(), repoUsage1);
    systemDiagnostics.setContentRepositoryStorageUsage(stringStorageUsageMap);
}

Example 5 with StorageUsage

use of org.apache.nifi.diagnostics.StorageUsage in project nifi-minifi by apache.

the class StatusConfigReporterTest method addFlowFileRepoToSystemDiagnostics.

private void addFlowFileRepoToSystemDiagnostics(SystemDiagnostics systemDiagnostics) {
    StorageUsage repoUsage = new StorageUsage();
    repoUsage.setFreeSpace(30);
    repoUsage.setTotalSpace(100);
    repoUsage.setIdentifier("FlowFile repo");
    systemDiagnostics.setFlowFileRepositoryStorageUsage(repoUsage);
}