Examples with AlluxioProperties alluxio.conf.AlluxioProperties used on opensource projects

Example 1 with AlluxioProperties

use of alluxio.conf.AlluxioProperties in project alluxio by Alluxio.

the class HadoopUtils method toAlluxioConf.

/**
 * @param conf Hadoop conf
 * @return Alluxio configuration merged from Hadoop conf
 */
public static AlluxioConfiguration toAlluxioConf(Configuration conf) {
    // Take hadoop configuration to merge to Alluxio configuration
    Map<String, Object> hadoopConfProperties = HadoopConfigurationUtils.getConfigurationFromHadoop(conf);
    AlluxioProperties alluxioProps = ConfigurationUtils.defaults();
    // Merge relevant Hadoop configuration into Alluxio's configuration.
    alluxioProps.merge(hadoopConfProperties, Source.RUNTIME);
    // Creating a new instanced configuration from an AlluxioProperties object isn't expensive.
    return new InstancedConfiguration(alluxioProps);
}

Example 2 with AlluxioProperties

use of alluxio.conf.AlluxioProperties in project alluxio by Alluxio.

the class ConfigurationUtils method merge.

/**
 * Merges the current configuration properties with new properties. If a property exists
 * both in the new and current configuration, the one from the new configuration wins if
 * its priority is higher or equal than the existing one.
 *
 * @param conf the base configuration
 * @param properties the source {@link Properties} to be merged
 * @param source the source of the the properties (e.g., system property, default and etc)
 * @return a new configuration representing the merged properties
 */
public static AlluxioConfiguration merge(AlluxioConfiguration conf, Map<?, ?> properties, Source source) {
    AlluxioProperties props = conf.copyProperties();
    props.merge(properties, source);
    return new InstancedConfiguration(props);
}

Example 3 with AlluxioProperties

use of alluxio.conf.AlluxioProperties in project alluxio by Alluxio.

the class ConfigurationUtils method getPathConf.

/**
 * Loads the path level configuration from the get configuration response.
 *
 * Only client scope properties will be loaded.
 *
 * @param response the get configuration RPC response
 * @param clusterConf cluster level configuration
 * @return the loaded path level configuration
 */
public static PathConfiguration getPathConf(GetConfigurationPResponse response, AlluxioConfiguration clusterConf) {
    String clientVersion = clusterConf.getString(PropertyKey.VERSION);
    LOG.debug("Alluxio client (version {}) is trying to load path level configurations", clientVersion);
    Map<String, AlluxioConfiguration> pathConfs = new HashMap<>();
    response.getPathConfigsMap().forEach((path, conf) -> {
        Properties props = filterAndLoadProperties(conf.getPropertiesList(), Scope.CLIENT, (key, value) -> String.format("Loading property: %s (%s) -> %s for path %s", key, key.getScope(), value, path));
        AlluxioProperties properties = new AlluxioProperties();
        properties.merge(props, Source.PATH_DEFAULT);
        pathConfs.put(path, new InstancedConfiguration(properties, true));
    });
    LOG.debug("Alluxio client has loaded path level configurations");
    return PathConfiguration.create(pathConfs);
}

Example 4 with AlluxioProperties

use of alluxio.conf.AlluxioProperties in project alluxio by Alluxio.

the class ConfigurationUtils method reloadProperties.

/**
 * Reloads site properties from disk.
 */
public static void reloadProperties() {
    synchronized (DEFAULT_PROPERTIES_LOCK) {
        // Step1: bootstrap the configuration. This is necessary because we need to resolve alluxio
        // .home (likely to be in system properties) to locate the conf dir to search for the site
        // property file.
        AlluxioProperties properties = new AlluxioProperties();
        InstancedConfiguration conf = new InstancedConfiguration(properties);
        // Can't directly pass System.getProperties() because it is not thread-safe
        // This can cause a ConcurrentModificationException when merging.
        Properties sysProps = new Properties();
        System.getProperties().stringPropertyNames().forEach(key -> sysProps.setProperty(key, System.getProperty(key)));
        properties.merge(sysProps, Source.SYSTEM_PROPERTY);
        // whether in test mode by default properties and system properties (via getBoolean).
        if (conf.getBoolean(PropertyKey.TEST_MODE)) {
            conf.validate();
            sDefaultProperties = properties;
            return;
        }
        // we are not in test mode, load site properties
        String confPaths = conf.getString(PropertyKey.SITE_CONF_DIR);
        String[] confPathList = confPaths.split(",");
        String sitePropertyFile = ConfigurationUtils.searchPropertiesFile(Constants.SITE_PROPERTIES, confPathList);
        Properties siteProps = null;
        if (sitePropertyFile != null) {
            siteProps = loadPropertiesFromFile(sitePropertyFile);
            sSourcePropertyFile = sitePropertyFile;
        } else {
            URL resource = ConfigurationUtils.class.getClassLoader().getResource(Constants.SITE_PROPERTIES);
            if (resource != null) {
                siteProps = loadPropertiesFromResource(resource);
                if (siteProps != null) {
                    sSourcePropertyFile = resource.getPath();
                }
            }
        }
        properties.merge(siteProps, Source.siteProperty(sSourcePropertyFile));
        conf.validate();
        sDefaultProperties = properties;
    }
}

Example 5 with AlluxioProperties

use of alluxio.conf.AlluxioProperties in project alluxio by Alluxio.

the class AbstractFileSystem method initialize.

/**
 * Initialize the {@link alluxio.hadoop.FileSystem}.
 * @param uri file system Uri
 * @param conf hadoop configuration
 * @param alluxioConfiguration [optional] alluxio configuration
 * @throws IOException
 */
@VisibleForTesting
public synchronized void initialize(URI uri, org.apache.hadoop.conf.Configuration conf, @Nullable AlluxioConfiguration alluxioConfiguration) throws IOException {
    // Validates scheme and authority of FS Uri.
    validateFsUri(uri);
    super.initialize(uri, conf);
    LOG.debug("initialize({}, {}). Connecting to Alluxio", uri, conf);
    HadoopUtils.addSwiftCredentials(conf);
    setConf(conf);
    // HDFS doesn't allow the authority to be empty; it must be "/" instead.
    String authority = uri.getAuthority() == null ? "/" : uri.getAuthority();
    mAlluxioHeader = getFsScheme(uri) + "://" + authority;
    // Set the statistics member. Use mStatistics instead of the parent class's variable.
    mStatistics = statistics;
    mUri = URI.create(mAlluxioHeader);
    // take the URI properties, hadoop configuration, and given Alluxio configuration and merge
    // all three into a single object.
    Map<String, Object> uriConfProperties = getConfigurationFromUri(uri, conf);
    Map<String, Object> hadoopConfProperties = HadoopConfigurationUtils.getConfigurationFromHadoop(conf);
    LOG.info("Creating Alluxio configuration from Hadoop configuration {}, uri configuration {}", hadoopConfProperties, uriConfProperties);
    AlluxioProperties alluxioProps = (alluxioConfiguration != null) ? alluxioConfiguration.copyProperties() : ConfigurationUtils.defaults();
    // Merge relevant Hadoop configuration into Alluxio's configuration.
    alluxioProps.merge(hadoopConfProperties, Source.RUNTIME);
    // Merge relevant connection details in the URI with the highest priority
    alluxioProps.merge(uriConfProperties, Source.RUNTIME);
    // Creating a new instanced configuration from an AlluxioProperties object isn't expensive.
    mAlluxioConf = new InstancedConfiguration(alluxioProps);
    mAlluxioConf.validate();
    if (mFileSystem != null) {
        return;
    }
    Subject subject = getHadoopSubject();
    LOG.debug("Using Hadoop subject: {}", subject);
    LOG.info("Initializing filesystem with connect details {}", Factory.getConnectDetails(mAlluxioConf));
    // Create FileSystem for accessing Alluxio.
    // Disable URI validation for non-Alluxio schemes.
    boolean enableUriValidation = (uri.getScheme() == null) || uri.getScheme().equals(Constants.SCHEME);
    mFileSystem = FileSystem.Factory.create(ClientContext.create(subject, mAlluxioConf).setUriValidationEnabled(enableUriValidation));
}