Examples with PropertyKey alluxio.conf.PropertyKey used on opensource projects

Example 41 with PropertyKey

use of alluxio.conf.PropertyKey in project alluxio by Alluxio.

the class ConfigurationUtils method getJobMasterEmbeddedJournalAddresses.

/**
 * @param conf configuration
 * @return the embedded journal addresses to use for the job master
 */
public static List<InetSocketAddress> getJobMasterEmbeddedJournalAddresses(AlluxioConfiguration conf) {
    PropertyKey jobMasterProperty = PropertyKey.JOB_MASTER_EMBEDDED_JOURNAL_ADDRESSES;
    if (conf.isSet(jobMasterProperty)) {
        return parseInetSocketAddresses(conf.getList(jobMasterProperty, ","));
    }
    // Fall back on using the master embedded journal addresses, with the job master port.
    PropertyKey masterProperty = PropertyKey.MASTER_EMBEDDED_JOURNAL_ADDRESSES;
    int jobRaftPort = NetworkAddressUtils.getPort(ServiceType.JOB_MASTER_RAFT, conf);
    if (conf.isSet(masterProperty)) {
        return overridePort(getMasterEmbeddedJournalAddresses(conf), jobRaftPort);
    }
    // Fall back on job_master_hostname:job_master_raft_port.
    return Arrays.asList(NetworkAddressUtils.getConnectAddress(ServiceType.JOB_MASTER_RAFT, conf));
}

Example 42 with PropertyKey

use of alluxio.conf.PropertyKey in project alluxio by Alluxio.

the class ConfigurationUtils method filterAndLoadProperties.

/**
 * Filters and loads properties with a certain scope from the property list returned by grpc.
 * The given scope should only be {@link Scope#WORKER} or {@link Scope#CLIENT}.
 *
 * @param properties the property list returned by grpc
 * @param scope the scope to filter the received property list
 * @param logMessage a function with key and value as parameter and returns debug log message
 * @return the loaded properties
 */
private static Properties filterAndLoadProperties(List<ConfigProperty> properties, Scope scope, BiFunction<PropertyKey, String, String> logMessage) {
    Properties props = new Properties();
    for (ConfigProperty property : properties) {
        String name = property.getName();
        // TODO(binfan): support propagating unsetting properties from master
        if (PropertyKey.isValid(name) && property.hasValue()) {
            PropertyKey key = PropertyKey.fromString(name);
            if (!GrpcUtils.contains(key.getScope(), scope)) {
                // Only propagate properties contains the target scope
                continue;
            }
            String value = property.getValue();
            props.put(key, value);
            LOG.debug(logMessage.apply(key, value));
        }
    }
    return props;
}

Example 43 with PropertyKey

use of alluxio.conf.PropertyKey in project alluxio by Alluxio.

the class StorageSpaceValidationTask method validateImpl.

@Override
public ValidationTaskResult validateImpl(Map<String, String> optionsMap) {
    StringBuilder msg = new StringBuilder();
    StringBuilder advice = new StringBuilder();
    int numLevel = mConf.getInt(PropertyKey.WORKER_TIERED_STORE_LEVELS);
    boolean success = true;
    for (int level = 0; level < numLevel; level++) {
        PropertyKey tierAliasConf = PropertyKey.Template.WORKER_TIERED_STORE_LEVEL_ALIAS.format(level);
        String alias = mConf.getString(tierAliasConf);
        PropertyKey tierDirPathConf = PropertyKey.Template.WORKER_TIERED_STORE_LEVEL_DIRS_PATH.format(level);
        String[] dirPaths = mConf.getString(tierDirPathConf).split(",");
        PropertyKey tierDirCapacityConf = PropertyKey.Template.WORKER_TIERED_STORE_LEVEL_DIRS_QUOTA.format(level);
        String rawDirQuota = mConf.getString(tierDirCapacityConf);
        if (rawDirQuota.isEmpty()) {
            msg.append(String.format("Tier %d: Quota cannot be empty.%n", level));
            advice.append(String.format("Please check your setting for %s.%n", tierDirCapacityConf));
            return new ValidationTaskResult(ValidationUtils.State.FAILED, getName(), msg.toString(), advice.toString());
        }
        String[] dirQuotas = rawDirQuota.split(",");
        try {
            Map<String, MountedStorage> storageMap = new HashMap<>();
            File file = new File(dirPaths[0]);
            if (dirPaths.length == 1 && alias.equals(Constants.MEDIUM_MEM) && !file.exists()) {
                msg.append(String.format("RAM disk is not mounted at %s, skip validation.%n", dirPaths[0]));
                continue;
            }
            boolean hasRamfsLocation = false;
            for (int i = 0; i < dirPaths.length; i++) {
                int index = i >= dirQuotas.length ? dirQuotas.length - 1 : i;
                if (ShellUtils.isMountingPoint(dirPaths[i], new String[] { "ramfs" })) {
                    msg.append(String.format("ramfs mounted at %s does not report space information," + " skip validation.%n", dirPaths[i]));
                    hasRamfsLocation = true;
                    break;
                }
                long quota = FormatUtils.parseSpaceSize(dirQuotas[index]);
                success &= addDirectoryInfo(dirPaths[i], quota, storageMap);
            }
            if (hasRamfsLocation) {
                continue;
            }
            for (Map.Entry<String, MountedStorage> storageEntry : storageMap.entrySet()) {
                MountedStorage storage = storageEntry.getValue();
                long quota = storage.getDesiredQuotaSizeBytes();
                long used = storage.getUsedTieredStorageSizeBytes();
                long available = storage.getAvailableSizeBytes();
                StringBuilder builder = new StringBuilder();
                for (Map.Entry<String, Long> directoryQuota : storage.getDirectoryQuotas().entrySet()) {
                    builder.append(String.format("- Quota for %s: %s%n", directoryQuota.getKey(), FormatUtils.getSizeFromBytes(directoryQuota.getValue())));
                }
                if (quota > used + available) {
                    msg.append(String.format("Tier %d: Not enough space on %s. %n" + "Total desired quota: %s%n" + "%s" + "Used in tiered storage: %s%n" + "Available: %s (Additional %s free space required).%n", level, storageEntry.getKey(), FormatUtils.getSizeFromBytes(quota), builder, FormatUtils.getSizeFromBytes(used), FormatUtils.getSizeFromBytes(available), FormatUtils.getSizeFromBytes(quota - used - available)));
                    advice.append(String.format("Please check your quota setting for tier %s.%n", level));
                    success = false;
                }
            }
        } catch (IOException e) {
            msg.append(String.format("Tier %d: Unable to validate available space - %s.%n", level, e.getMessage()));
            msg.append(ValidationUtils.getErrorInfo(e));
            advice.append(String.format("Please check your path for tier %s.%n", level));
            success = false;
        }
    }
    ValidationUtils.State state = success ? ValidationUtils.State.OK : ValidationUtils.State.WARNING;
    return new ValidationTaskResult(state, getName(), msg.toString(), advice.toString());
}

Example 44 with PropertyKey

use of alluxio.conf.PropertyKey in project alluxio by Alluxio.

the class ClusterConfConsistencyValidationTask method validateImpl.

@Override
protected ValidationTaskResult validateImpl(Map<String, String> optionMap) throws InterruptedException {
    StringBuilder msg = new StringBuilder();
    StringBuilder advice = new StringBuilder();
    Set<String> masters = ConfigurationUtils.getMasterHostnames(mConf);
    Set<String> workers = ConfigurationUtils.getWorkerHostnames(mConf);
    Set<String> nodes = Sets.union(masters, workers);
    Map<String, Properties> allProperties = new HashMap<>();
    Set<String> propertyNames = new HashSet<>();
    if (masters.isEmpty()) {
        msg.append(String.format("No master nodes specified in %s/masters file. ", mConf.get(PropertyKey.CONF_DIR)));
        advice.append(String.format("Please configure %s to contain the master node hostnames. ", mConf.get(PropertyKey.CONF_DIR)));
        return new ValidationTaskResult(ValidationUtils.State.WARNING, getName(), msg.toString(), advice.toString());
    }
    if (workers.isEmpty()) {
        msg.append(String.format("No worker nodes specified in %s/workers file. ", mConf.get(PropertyKey.CONF_DIR)));
        advice.append(String.format("Please configure %s to contain the worker node hostnames. ", mConf.get(PropertyKey.CONF_DIR)));
        return new ValidationTaskResult(ValidationUtils.State.WARNING, getName(), msg.toString(), advice.toString());
    }
    ValidationUtils.State state = ValidationUtils.State.OK;
    for (String node : nodes) {
        try {
            Properties props = getNodeConf(node);
            allProperties.put(node, props);
            propertyNames.addAll(props.stringPropertyNames());
        } catch (IOException e) {
            System.err.format("Unable to retrieve configuration for %s: %s.", node, e.getMessage());
            msg.append(String.format("Unable to retrieve configuration for %s: %s.", node, e.getMessage()));
            advice.append(String.format("Please check the connection from node %s. ", node));
            state = ValidationUtils.State.FAILED;
        // Check all nodes before returning
        }
    }
    for (String propertyName : propertyNames) {
        if (!PropertyKey.isValid(propertyName)) {
            continue;
        }
        PropertyKey propertyKey = PropertyKey.fromString(propertyName);
        PropertyKey.ConsistencyCheckLevel level = propertyKey.getConsistencyLevel();
        if (level == PropertyKey.ConsistencyCheckLevel.IGNORE) {
            continue;
        }
        Scope scope = propertyKey.getScope();
        Set<String> targetNodes = ImmutableSet.of();
        if (GrpcUtils.contains(scope, Scope.MASTER)) {
            targetNodes = masters;
        }
        if (GrpcUtils.contains(scope, Scope.WORKER)) {
            targetNodes = Sets.union(targetNodes, workers);
        }
        if (targetNodes.size() < 2) {
            continue;
        }
        String baseNode = null;
        String baseValue = null;
        boolean isConsistent = true;
        String errLabel;
        ValidationUtils.State errLevel;
        switch(level) {
            case ENFORCE:
                errLabel = "Error";
                errLevel = ValidationUtils.State.FAILED;
                break;
            case WARN:
                errLabel = "Warning";
                errLevel = ValidationUtils.State.WARNING;
                break;
            default:
                msg.append(String.format("Error: Consistency check level \"%s\" for property \"%s\" is invalid.%n", level.name(), propertyName));
                advice.append(String.format("Please check property %s.%n", propertyName));
                state = ValidationUtils.State.FAILED;
                continue;
        }
        for (String remoteNode : targetNodes) {
            if (baseNode == null) {
                baseNode = remoteNode;
                Properties baselineProps = allProperties.get(baseNode);
                baseValue = baselineProps.getProperty(propertyName);
                continue;
            }
            String remoteValue = allProperties.get(remoteNode).getProperty(propertyName);
            if (!StringUtils.equals(remoteValue, baseValue)) {
                msg.append(String.format("%s: Property \"%s\" is inconsistent between node %s and %s.%n", errLabel, propertyName, baseNode, remoteNode));
                msg.append(String.format(" %s: %s%n %s: %s%n", baseNode, Objects.toString(baseValue, "not set"), remoteNode, Objects.toString(remoteValue, "not set")));
                advice.append(String.format("Please check your settings for property %s on %s and %s.%n", propertyName, baseNode, remoteNode));
                isConsistent = false;
            }
        }
        if (!isConsistent) {
            state = state == ValidationUtils.State.FAILED ? ValidationUtils.State.FAILED : errLevel;
        }
    }
    return new ValidationTaskResult(state, getName(), msg.toString(), advice.toString());
}

Example 45 with PropertyKey

use of alluxio.conf.PropertyKey in project alluxio by Alluxio.

the class ShowCommand method run.

@Override
public int run(CommandLine cl) throws IOException {
    String targetPath = cl.getArgs()[0];
    Configuration configuration = mMetaConfigClient.getConfiguration(GetConfigurationPOptions.getDefaultInstance());
    if (cl.hasOption(ALL_OPTION_NAME)) {
        Map<String, AlluxioConfiguration> pathConfMap = new HashMap<>();
        configuration.getPathConf().forEach((path, conf) -> {
            AlluxioProperties properties = new AlluxioProperties();
            conf.forEach(property -> {
                PropertyKey key = PropertyKey.fromString(property.getName());
                properties.set(key, property.getValue());
            });
            pathConfMap.put(path, new InstancedConfiguration(properties));
        });
        PathConfiguration pathConf = PathConfiguration.create(pathConfMap);
        AlluxioURI targetUri = new AlluxioURI(targetPath);
        List<PropertyKey> propertyKeys = new ArrayList<>(pathConf.getPropertyKeys(targetUri));
        propertyKeys.sort(Comparator.comparing(PropertyKey::getName));
        propertyKeys.forEach(key -> {
            pathConf.getConfiguration(targetUri, key).ifPresent(conf -> {
                mPrintStream.println(format(key.getName(), conf.get(key)));
            });
        });
    } else if (configuration.getPathConf().containsKey(targetPath)) {
        List<Property> properties = configuration.getPathConf().get(targetPath);
        properties.sort(Comparator.comparing(Property::getName));
        properties.forEach(property -> {
            mPrintStream.println(format(property.getName(), property.getValue()));
        });
    }
    return 0;
}