Examples with Scope alluxio.grpc.Scope used on opensource projects

Example 1 with Scope

use of alluxio.grpc.Scope in project alluxio by Alluxio.

the class ConfigurationUtils method getConfiguration.

/**
 * Gets all configuration properties filtered by the specified scope.
 *
 * @param conf the configuration to use
 * @param scope the scope to filter by
 * @return the properties
 */
public static List<ConfigProperty> getConfiguration(AlluxioConfiguration conf, Scope scope) {
    ConfigurationValueOptions useRawDisplayValue = ConfigurationValueOptions.defaults().useDisplayValue(true);
    List<ConfigProperty> configs = new ArrayList<>();
    List<PropertyKey> selectedKeys = conf.keySet().stream().filter(key -> GrpcUtils.contains(key.getScope(), scope)).filter(key -> key.isValid(key.getName())).collect(toList());
    for (PropertyKey key : selectedKeys) {
        ConfigProperty.Builder configProp = ConfigProperty.newBuilder().setName(key.getName()).setSource(conf.getSource(key).toString());
        if (conf.isSet(key)) {
            configProp.setValue(String.valueOf(conf.get(key, useRawDisplayValue)));
        }
        configs.add(configProp.build());
    }
    return configs;
}

Example 2 with Scope

use of alluxio.grpc.Scope in project alluxio by Alluxio.

the class ServerConfigurationChecker method regenerateReport.

/**
 * Checks the server-side configurations and records the check results.
 */
public synchronized void regenerateReport() {
    // Generate the configuration map from master and worker configuration records
    Map<PropertyKey, Map<Optional<String>, List<String>>> confMap = generateConfMap();
    // Update the errors and warnings configuration
    Map<Scope, List<InconsistentProperty>> confErrors = new HashMap<>();
    Map<Scope, List<InconsistentProperty>> confWarns = new HashMap<>();
    for (Map.Entry<PropertyKey, Map<Optional<String>, List<String>>> entry : confMap.entrySet()) {
        if (entry.getValue().size() >= 2) {
            PropertyKey key = entry.getKey();
            InconsistentProperty inconsistentProperty = new InconsistentProperty().setName(key.getName()).setValues(entry.getValue());
            Scope scope = key.getScope().equals(Scope.ALL) ? Scope.SERVER : key.getScope();
            if (entry.getKey().getConsistencyLevel().equals(ConsistencyCheckLevel.ENFORCE)) {
                confErrors.putIfAbsent(scope, new ArrayList<>());
                confErrors.get(scope).add(inconsistentProperty);
            } else {
                confWarns.putIfAbsent(scope, new ArrayList<>());
                confWarns.get(scope).add(inconsistentProperty);
            }
        }
    }
    // Update configuration status
    ConfigStatus status = confErrors.values().stream().anyMatch(a -> a.size() > 0) ? ConfigStatus.FAILED : confWarns.values().stream().anyMatch(a -> a.size() > 0) ? ConfigStatus.WARN : ConfigStatus.PASSED;
    if (!status.equals(mConfigCheckReport.getConfigStatus())) {
        logConfigReport();
    }
    mConfigCheckReport = new ConfigCheckReport(confErrors, confWarns, status);
}

Example 3 with Scope

use of alluxio.grpc.Scope in project alluxio by Alluxio.

the class ClusterConfConsistencyValidationTask method validateImpl.

@Override
protected ValidationTaskResult validateImpl(Map<String, String> optionMap) throws InterruptedException {
    StringBuilder msg = new StringBuilder();
    StringBuilder advice = new StringBuilder();
    Set<String> masters = ConfigurationUtils.getMasterHostnames(mConf);
    Set<String> workers = ConfigurationUtils.getWorkerHostnames(mConf);
    Set<String> nodes = Sets.union(masters, workers);
    Map<String, Properties> allProperties = new HashMap<>();
    Set<String> propertyNames = new HashSet<>();
    if (masters.isEmpty()) {
        msg.append(String.format("No master nodes specified in %s/masters file. ", mConf.get(PropertyKey.CONF_DIR)));
        advice.append(String.format("Please configure %s to contain the master node hostnames. ", mConf.get(PropertyKey.CONF_DIR)));
        return new ValidationTaskResult(ValidationUtils.State.WARNING, getName(), msg.toString(), advice.toString());
    }
    if (workers.isEmpty()) {
        msg.append(String.format("No worker nodes specified in %s/workers file. ", mConf.get(PropertyKey.CONF_DIR)));
        advice.append(String.format("Please configure %s to contain the worker node hostnames. ", mConf.get(PropertyKey.CONF_DIR)));
        return new ValidationTaskResult(ValidationUtils.State.WARNING, getName(), msg.toString(), advice.toString());
    }
    ValidationUtils.State state = ValidationUtils.State.OK;
    for (String node : nodes) {
        try {
            Properties props = getNodeConf(node);
            allProperties.put(node, props);
            propertyNames.addAll(props.stringPropertyNames());
        } catch (IOException e) {
            System.err.format("Unable to retrieve configuration for %s: %s.", node, e.getMessage());
            msg.append(String.format("Unable to retrieve configuration for %s: %s.", node, e.getMessage()));
            advice.append(String.format("Please check the connection from node %s. ", node));
            state = ValidationUtils.State.FAILED;
        // Check all nodes before returning
        }
    }
    for (String propertyName : propertyNames) {
        if (!PropertyKey.isValid(propertyName)) {
            continue;
        }
        PropertyKey propertyKey = PropertyKey.fromString(propertyName);
        PropertyKey.ConsistencyCheckLevel level = propertyKey.getConsistencyLevel();
        if (level == PropertyKey.ConsistencyCheckLevel.IGNORE) {
            continue;
        }
        Scope scope = propertyKey.getScope();
        Set<String> targetNodes = ImmutableSet.of();
        if (GrpcUtils.contains(scope, Scope.MASTER)) {
            targetNodes = masters;
        }
        if (GrpcUtils.contains(scope, Scope.WORKER)) {
            targetNodes = Sets.union(targetNodes, workers);
        }
        if (targetNodes.size() < 2) {
            continue;
        }
        String baseNode = null;
        String baseValue = null;
        boolean isConsistent = true;
        String errLabel;
        ValidationUtils.State errLevel;
        switch(level) {
            case ENFORCE:
                errLabel = "Error";
                errLevel = ValidationUtils.State.FAILED;
                break;
            case WARN:
                errLabel = "Warning";
                errLevel = ValidationUtils.State.WARNING;
                break;
            default:
                msg.append(String.format("Error: Consistency check level \"%s\" for property \"%s\" is invalid.%n", level.name(), propertyName));
                advice.append(String.format("Please check property %s.%n", propertyName));
                state = ValidationUtils.State.FAILED;
                continue;
        }
        for (String remoteNode : targetNodes) {
            if (baseNode == null) {
                baseNode = remoteNode;
                Properties baselineProps = allProperties.get(baseNode);
                baseValue = baselineProps.getProperty(propertyName);
                continue;
            }
            String remoteValue = allProperties.get(remoteNode).getProperty(propertyName);
            if (!StringUtils.equals(remoteValue, baseValue)) {
                msg.append(String.format("%s: Property \"%s\" is inconsistent between node %s and %s.%n", errLabel, propertyName, baseNode, remoteNode));
                msg.append(String.format(" %s: %s%n %s: %s%n", baseNode, Objects.toString(baseValue, "not set"), remoteNode, Objects.toString(remoteValue, "not set")));
                advice.append(String.format("Please check your settings for property %s on %s and %s.%n", propertyName, baseNode, remoteNode));
                isConsistent = false;
            }
        }
        if (!isConsistent) {
            state = state == ValidationUtils.State.FAILED ? ValidationUtils.State.FAILED : errLevel;
        }
    }
    return new ValidationTaskResult(state, getName(), msg.toString(), advice.toString());
}

Example 4 with Scope

use of alluxio.grpc.Scope in project alluxio by Alluxio.

the class ConfigurationCommand method run.

/**
 * Runs doctor configuration command.
 *
 * @return 0 on success, 1 otherwise
 */
public int run() throws IOException {
    ConfigCheckReport report = mMetaMasterClient.getConfigReport();
    ConfigStatus configStatus = report.getConfigStatus();
    if (configStatus == ConfigStatus.PASSED) {
        // No errors or warnings to show
        mPrintStream.println("No server-side configuration errors or warnings.");
        return 0;
    }
    Map<Scope, List<InconsistentProperty>> errors = report.getConfigErrors();
    if (errors.size() != 0) {
        mPrintStream.println("Server-side configuration errors " + "(those properties are required to be identical): ");
        printInconsistentProperties(errors);
    }
    Map<Scope, List<InconsistentProperty>> warnings = report.getConfigWarns();
    if (warnings.size() != 0) {
        mPrintStream.println("\nServer-side configuration warnings " + "(those properties are recommended to be identical): ");
        printInconsistentProperties(warnings);
    }
    return 0;
}

Example 5 with Scope

use of alluxio.grpc.Scope in project alluxio by Alluxio.

the class ConfigCheckerIntegrationTest method unsetVsSet.

@Test
public void unsetVsSet() throws Exception {
    Map<Integer, Map<PropertyKey, String>> masterProperties = ImmutableMap.of(1, ImmutableMap.of(PropertyKey.MASTER_MOUNT_TABLE_ROOT_OPTION, "option"));
    mCluster = MultiProcessCluster.newBuilder(PortCoordination.CONFIG_CHECKER_UNSET_VS_SET).setClusterName("ConfigCheckerUnsetVsSet").setNumMasters(2).setNumWorkers(0).setMasterProperties(masterProperties).build();
    mCluster.start();
    ConfigCheckReport report = getReport();
    Map<Scope, List<InconsistentProperty>> errors = report.getConfigErrors();
    assertTrue(errors.containsKey(Scope.MASTER));
    if (mCluster.getDeployMode().equals(DeployMode.ZOOKEEPER_HA)) {
        assertEquals(1, errors.get(Scope.MASTER).size());
        InconsistentProperty property = errors.get(Scope.MASTER).get(0);
        assertEquals(PropertyKey.MASTER_MOUNT_TABLE_ROOT_OPTION.getName(), property.getName());
        assertTrue(property.getValues().containsKey(Optional.of("option")));
        assertTrue(property.getValues().containsKey(Optional.empty()));
    } else {
        // When using embedded journal, the journal paths are different
        assertEquals(2, errors.get(Scope.MASTER).size());
        assertThat(report.getConfigErrors().toString(), CoreMatchers.containsString(PropertyKey.MASTER_MOUNT_TABLE_ROOT_OPTION.getName()));
    }
    mCluster.notifySuccess();
}