Examples with ConfigCheckReport alluxio.wire.ConfigCheckReport used on opensource projects

Example 1 with ConfigCheckReport

use of alluxio.wire.ConfigCheckReport in project alluxio by Alluxio.

the class ConfigCheckerIntegrationTest method multiMasters.

@Test
public void multiMasters() throws Exception {
    PropertyKey key = PropertyKey.MASTER_JOURNAL_FLUSH_TIMEOUT_MS;
    Map<Integer, Map<PropertyKey, String>> masterProperties = generatePropertyWithDifferentValues(TEST_NUM_MASTERS, key);
    mCluster = MultiProcessCluster.newBuilder(PortCoordination.CONFIG_CHECKER_MULTI_MASTERS).setClusterName("ConfigCheckerMultiMastersTest").setNumMasters(TEST_NUM_MASTERS).setNumWorkers(0).setMasterProperties(masterProperties).build();
    mCluster.start();
    ConfigCheckReport report = getReport();
    // When using embedded journal, the journal paths are different
    assertEquals(mCluster.getDeployMode().equals(DeployMode.ZOOKEEPER_HA) ? ConfigStatus.WARN : ConfigStatus.FAILED, report.getConfigStatus());
    assertThat(report.getConfigWarns().toString(), CoreMatchers.containsString(key.getName()));
    mCluster.notifySuccess();
}

Example 2 with ConfigCheckReport

use of alluxio.wire.ConfigCheckReport in project alluxio by Alluxio.

the class ConfigCheckerIntegrationTest method multiNodes.

@Test
public void multiNodes() throws Exception {
    PropertyKey key = PropertyKey.UNDERFS_LISTING_LENGTH;
    // Prepare properties
    Map<Integer, Map<PropertyKey, String>> properties = generatePropertyWithDifferentValues(TEST_NUM_MASTERS + TEST_NUM_WORKERS, key);
    Map<Integer, Map<PropertyKey, String>> masterProperties = properties.entrySet().stream().filter(entry -> (entry.getKey() < TEST_NUM_MASTERS)).collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue));
    Map<Integer, Map<PropertyKey, String>> workerProperties = properties.entrySet().stream().filter(entry -> (entry.getKey() >= TEST_NUM_MASTERS)).collect(Collectors.toMap(entry -> entry.getKey() - TEST_NUM_MASTERS, Map.Entry::getValue));
    mCluster = MultiProcessCluster.newBuilder(PortCoordination.CONFIG_CHECKER_MULTI_NODES).setClusterName("ConfigCheckerMultiNodesTest").setNumMasters(TEST_NUM_MASTERS).setNumWorkers(TEST_NUM_WORKERS).setMasterProperties(masterProperties).setWorkerProperties(workerProperties).build();
    mCluster.start();
    ConfigCheckReport report = getReport();
    assertEquals(ConfigStatus.FAILED, report.getConfigStatus());
    assertThat(report.getConfigErrors().toString(), CoreMatchers.containsString(key.getName()));
    mCluster.notifySuccess();
}

Example 3 with ConfigCheckReport

use of alluxio.wire.ConfigCheckReport in project alluxio by Alluxio.

the class ConfigCheckerIntegrationTest method multiWorkers.

@Test
public void multiWorkers() throws Exception {
    PropertyKey key = PropertyKey.WORKER_FREE_SPACE_TIMEOUT;
    Map<Integer, Map<PropertyKey, String>> workerProperties = generatePropertyWithDifferentValues(TEST_NUM_WORKERS, key);
    mCluster = MultiProcessCluster.newBuilder(PortCoordination.CONFIG_CHECKER_MULTI_WORKERS).setClusterName("ConfigCheckerMultiWorkersTest").setNumMasters(1).setNumWorkers(TEST_NUM_WORKERS).setWorkerProperties(workerProperties).build();
    mCluster.start();
    ConfigCheckReport report = getReport();
    // The workers values of many directory related properties are different
    assertEquals(ConfigStatus.WARN, report.getConfigStatus());
    assertThat(report.getConfigWarns().toString(), CoreMatchers.containsString(key.getName()));
    mCluster.notifySuccess();
}

Example 4 with ConfigCheckReport

use of alluxio.wire.ConfigCheckReport in project alluxio by Alluxio.

the class ServerConfigurationCheckerTest method checkResults.

/**
 * Checks if the server-side configuration report is as expected.
 *
 * @param expectedErrorNum the expected error number
 * @param expectedWarnNum the expected warning number
 * @param expectedStatus the expected config check status
 */
private void checkResults(int expectedErrorNum, int expectedWarnNum, ConfigStatus expectedStatus) {
    mConfigChecker.regenerateReport();
    ConfigCheckReport report = mConfigChecker.getConfigCheckReport();
    assertEquals(expectedErrorNum, report.getConfigErrors().size());
    assertEquals(expectedWarnNum, report.getConfigWarns().size());
    assertEquals(expectedStatus, report.getConfigStatus());
}

Example 5 with ConfigCheckReport

use of alluxio.wire.ConfigCheckReport in project alluxio by Alluxio.

the class AlluxioMasterRestServiceHandler method getWebUIOverview.

/**
 * Gets Web UI overview page data.
 *
 * @return the response object
 */
@GET
@Path(WEBUI_OVERVIEW)
public Response getWebUIOverview() {
    return RestUtils.call(() -> {
        MasterWebUIOverview response = new MasterWebUIOverview();
        response.setDebug(ServerConfiguration.getBoolean(PropertyKey.DEBUG)).setMasterNodeAddress(mMasterProcess.getRpcAddress().toString()).setUptime(CommonUtils.convertMsToClockTime(System.currentTimeMillis() - mMetaMaster.getStartTimeMs())).setStartTime(CommonUtils.convertMsToDate(mMetaMaster.getStartTimeMs(), ServerConfiguration.getString(PropertyKey.USER_DATE_FORMAT_PATTERN))).setVersion(RuntimeConstants.VERSION).setLiveWorkerNodes(Integer.toString(mBlockMaster.getWorkerCount())).setCapacity(FormatUtils.getSizeFromBytes(mBlockMaster.getCapacityBytes())).setClusterId(mMetaMaster.getClusterID()).setReplicaBlockCount(Long.toString(mBlockMaster.getBlockReplicaCount())).setUniqueBlockCount(Long.toString(mBlockMaster.getUniqueBlockCount())).setTotalPath(Long.toString(mFileSystemMaster.getInodeCount())).setUsedCapacity(FormatUtils.getSizeFromBytes(mBlockMaster.getUsedBytes())).setFreeCapacity(FormatUtils.getSizeFromBytes(mBlockMaster.getCapacityBytes() - mBlockMaster.getUsedBytes()));
        ConfigCheckReport report = mMetaMaster.getConfigCheckReport();
        response.setConfigCheckStatus(report.getConfigStatus()).setConfigCheckErrors(report.getConfigErrors()).setConfigCheckWarns(report.getConfigWarns()).setConfigCheckErrorNum(report.getConfigErrors().values().stream().mapToInt(List::size).sum()).setConfigCheckWarnNum(report.getConfigWarns().values().stream().mapToInt(List::size).sum());
        StorageTierAssoc globalStorageTierAssoc = mBlockMaster.getGlobalStorageTierAssoc();
        List<StorageTierInfo> infosList = new ArrayList<>();
        Map<String, Long> totalBytesOnTiers = mBlockMaster.getTotalBytesOnTiers();
        Map<String, Long> usedBytesOnTiers = mBlockMaster.getUsedBytesOnTiers();
        for (int ordinal = 0; ordinal < globalStorageTierAssoc.size(); ordinal++) {
            String tierAlias = globalStorageTierAssoc.getAlias(ordinal);
            if (totalBytesOnTiers.containsKey(tierAlias) && totalBytesOnTiers.get(tierAlias) > 0) {
                StorageTierInfo info = new StorageTierInfo(tierAlias, totalBytesOnTiers.get(tierAlias), usedBytesOnTiers.get(tierAlias));
                infosList.add(info);
            }
        }
        response.setStorageTierInfos(infosList);
        MountPointInfo mountInfo;
        try {
            mountInfo = mFileSystemMaster.getDisplayMountPointInfo(new AlluxioURI(MountTable.ROOT));
            long capacityBytes = mountInfo.getUfsCapacityBytes();
            long usedBytes = mountInfo.getUfsUsedBytes();
            long freeBytes = -1;
            if (usedBytes >= 0 && capacityBytes >= usedBytes) {
                freeBytes = capacityBytes - usedBytes;
            }
            String totalSpace = "UNKNOWN";
            if (capacityBytes >= 0) {
                totalSpace = FormatUtils.getSizeFromBytes(capacityBytes);
            }
            response.setDiskCapacity(totalSpace);
            String usedSpace = "UNKNOWN";
            if (usedBytes >= 0) {
                usedSpace = FormatUtils.getSizeFromBytes(usedBytes);
            }
            response.setDiskUsedCapacity(usedSpace);
            String freeSpace = "UNKNOWN";
            if (freeBytes >= 0) {
                freeSpace = FormatUtils.getSizeFromBytes(freeBytes);
            }
            response.setDiskFreeCapacity(freeSpace);
        } catch (Throwable e) {
            response.setDiskCapacity("UNKNOWN").setDiskUsedCapacity("UNKNOWN").setDiskFreeCapacity("UNKNOWN");
        }
        mMetaMaster.getJournalSpaceMonitor().map(monitor -> response.setJournalDiskWarnings(monitor.getJournalDiskWarnings()));
        Gauge entriesSinceGauge = MetricsSystem.METRIC_REGISTRY.getGauges().get(MetricKey.MASTER_JOURNAL_ENTRIES_SINCE_CHECKPOINT.getName());
        Gauge lastCkPtGauge = MetricsSystem.METRIC_REGISTRY.getGauges().get(MetricKey.MASTER_JOURNAL_LAST_CHECKPOINT_TIME.getName());
        if (entriesSinceGauge != null && lastCkPtGauge != null) {
            long entriesSinceCkpt = (Long) entriesSinceGauge.getValue();
            long lastCkptTime = (Long) lastCkPtGauge.getValue();
            long timeSinceCkpt = System.currentTimeMillis() - lastCkptTime;
            boolean overThreshold = timeSinceCkpt > ServerConfiguration.getMs(PropertyKey.MASTER_WEB_JOURNAL_CHECKPOINT_WARNING_THRESHOLD_TIME);
            boolean passedThreshold = entriesSinceCkpt > ServerConfiguration.getLong(PropertyKey.MASTER_JOURNAL_CHECKPOINT_PERIOD_ENTRIES);
            if (passedThreshold && overThreshold) {
                String time = lastCkptTime > 0 ? ZonedDateTime.ofInstant(Instant.ofEpochMilli(lastCkptTime), ZoneOffset.UTC).format(DateTimeFormatter.ISO_INSTANT) : "N/A";
                String advice = ConfigurationUtils.isHaMode(ServerConfiguration.global()) ? "" : "It is recommended to use the fsadmin tool to checkpoint the journal. This will " + "prevent the master from serving requests while checkpointing.";
                response.setJournalCheckpointTimeWarning(String.format("Journal has not checkpointed in " + "a timely manner since passing the checkpoint threshold (%d/%d). Last checkpoint:" + " %s. %s", entriesSinceCkpt, ServerConfiguration.getLong(PropertyKey.MASTER_JOURNAL_CHECKPOINT_PERIOD_ENTRIES), time, advice));
            }
        }
        Gauge masterRoleIdGauge = MetricsSystem.METRIC_REGISTRY.getGauges().get(MetricKey.MASTER_ROLE_ID.getName());
        Gauge leaderIdGauge = MetricsSystem.METRIC_REGISTRY.getGauges().get(MetricKey.CLUSTER_LEADER_ID.getName());
        if (masterRoleIdGauge != null) {
            response.setMasterRole(RaftProtos.RaftPeerRole.forNumber((Integer) masterRoleIdGauge.getValue()).name());
        }
        if (leaderIdGauge != null) {
            response.setLeaderId((String) leaderIdGauge.getValue());
        }
        return response;
    }, ServerConfiguration.global());
}