Examples with UnderFileSystemConfiguration alluxio.underfs.UnderFileSystemConfiguration used on opensource projects

Example 6 with UnderFileSystemConfiguration

use of alluxio.underfs.UnderFileSystemConfiguration in project alluxio by Alluxio.

the class HdfsUnderFileSystemTest method before.

@Before
public final void before() throws Exception {
    UnderFileSystemConfiguration conf = UnderFileSystemConfiguration.defaults(ConfigurationTestUtils.defaults()).createMountSpecificConf(ImmutableMap.of("hadoop.security.group.mapping", "org.apache.hadoop.security.ShellBasedUnixGroupsMapping", "fs.hdfs.impl", PropertyKey.UNDERFS_HDFS_IMPL.getDefaultStringValue()));
    mHdfsUnderFileSystem = HdfsUnderFileSystem.createInstance(new AlluxioURI(mTemporaryFolder.getRoot().getAbsolutePath()), conf);
}

Example 7 with UnderFileSystemConfiguration

use of alluxio.underfs.UnderFileSystemConfiguration in project alluxio by Alluxio.

the class UfsJournalConfTest method emptyConfiguration.

@Test
public void emptyConfiguration() throws Exception {
    UnderFileSystemConfiguration conf = UfsJournal.getJournalUfsConf();
    Assert.assertTrue(conf.getMountSpecificConf().isEmpty());
}

Example 8 with UnderFileSystemConfiguration

use of alluxio.underfs.UnderFileSystemConfiguration in project alluxio by Alluxio.

the class AbfsUnderFileSystem method createAbfsConfiguration.

private static Configuration createAbfsConfiguration(UnderFileSystemConfiguration conf) {
    Configuration abfsConf = HdfsUnderFileSystem.createConfiguration(conf);
    boolean sharedKey = false;
    for (Map.Entry<String, Object> entry : conf.toMap().entrySet()) {
        String key = entry.getKey();
        Object value = entry.getValue();
        if (PropertyKey.Template.UNDERFS_ABFS_ACCOUNT_KEY.matches(key)) {
            abfsConf.set(key, (String) value);
            final String authTypeKey = key.replace(".key", ".auth.type");
            abfsConf.set(authTypeKey, "SharedKey");
            sharedKey = true;
            break;
        }
    }
    if (!sharedKey) {
        if (conf.isSet(PropertyKey.ABFS_CLIENT_ENDPOINT) && conf.isSet(PropertyKey.ABFS_CLIENT_ID) && conf.isSet(PropertyKey.ABFS_CLIENT_SECRET)) {
            abfsConf.set("fs.azure.account.auth.type", "OAuth");
            abfsConf.set("fs.azure.account.oauth.provider.type", "org.apache.hadoop.fs.azurebfs.oauth2.ClientCredsTokenProvider");
            abfsConf.set(PropertyKey.ABFS_CLIENT_ENDPOINT.getName(), conf.getString(PropertyKey.ABFS_CLIENT_ENDPOINT));
            abfsConf.set(PropertyKey.ABFS_CLIENT_ID.getName(), conf.getString(PropertyKey.ABFS_CLIENT_ID));
            abfsConf.set(PropertyKey.ABFS_CLIENT_SECRET.getName(), conf.getString(PropertyKey.ABFS_CLIENT_SECRET));
        }
    }
    return abfsConf;
}

Example 9 with UnderFileSystemConfiguration

use of alluxio.underfs.UnderFileSystemConfiguration in project alluxio by Alluxio.

the class WasbUnderFileSystem method createConfiguration.

/**
 * Prepares the configuration for this Wasb as an HDFS configuration.
 *
 * @param conf the configuration for this UFS
 * @param isSecure whether blob storage is using https
 * @return the created configuration
 */
public static Configuration createConfiguration(UnderFileSystemConfiguration conf, Boolean isSecure) {
    Configuration wasbConf = HdfsUnderFileSystem.createConfiguration(conf);
    for (Map.Entry<String, Object> entry : conf.toMap().entrySet()) {
        String key = entry.getKey();
        Object value = entry.getValue();
        if (PropertyKey.Template.UNDERFS_AZURE_ACCOUNT_KEY.matches(key)) {
            wasbConf.set(key, (String) value);
        }
    }
    if (isSecure) {
        wasbConf.set("fs.AbstractFileSystem.wasbs.impl", "org.apache.hadoop.fs.azure.Wasbs");
        wasbConf.set("fs.wasbs.impl", "org.apache.hadoop.fs.azure.NativeAzureFileSystem");
    } else {
        wasbConf.set("fs.AbstractFileSystem.wasb.impl", "org.apache.hadoop.fs.azure.Wasb");
        wasbConf.set("fs.wasb.impl", "org.apache.hadoop.fs.azure.NativeAzureFileSystem");
    }
    return wasbConf;
}

Example 10 with UnderFileSystemConfiguration

use of alluxio.underfs.UnderFileSystemConfiguration in project alluxio by Alluxio.

the class ValidateHdfsMount method main.

/**
 * The entrance.
 *
 * @param args command line arguments
 */
public static void main(String[] args) throws Exception {
    CommandLineParser parser = new DefaultParser();
    CommandLine cmd = null;
    try {
        cmd = parser.parse(OPTIONS, args, true);
    } catch (ParseException e) {
        printHelp(String.format("Failed to parse arguments %s%n", Arrays.toString(args)));
        System.exit(1);
    }
    if (cmd.hasOption(HELP_OPTION.getLongOpt())) {
        printHelp("Showing usage for the command");
        System.exit(0);
    }
    args = cmd.getArgs();
    if (args.length < 1) {
        printHelp("Need at least 1 argument for <hdfsURI>!");
        System.exit(1);
    }
    String ufsPath = args[0];
    InstancedConfiguration conf = InstancedConfiguration.defaults();
    // Merge options from the command line option
    UnderFileSystemConfiguration ufsConf = UnderFileSystemConfiguration.defaults(conf);
    if (cmd.hasOption(READONLY_OPTION.getLongOpt())) {
        ufsConf.setReadOnly(true);
    }
    if (cmd.hasOption(SHARED_OPTION.getLongOpt())) {
        ufsConf.setShared(true);
    }
    if (cmd.hasOption(OPTION_OPTION.getLongOpt())) {
        Properties properties = cmd.getOptionProperties(OPTION_OPTION.getLongOpt());
        ufsConf.merge(properties, Source.MOUNT_OPTION);
        LOG.debug("Options from cmdline: {}", properties);
    }
    ValidationToolRegistry registry = new ValidationToolRegistry(new InstancedConfiguration(ConfigurationUtils.defaults()));
    // Load hdfs validation tool from alluxio lib directory
    registry.refresh();
    Map<Object, Object> configMap = new HashMap<>();
    configMap.put(ValidationConfig.UFS_PATH, ufsPath);
    configMap.put(ValidationConfig.UFS_CONFIG, ufsConf);
    ValidationTool tests = registry.create(ValidationConfig.HDFS_TOOL_TYPE, configMap);
    String result = ValidationTool.toJson(ValidationTool.convertResults(tests.runAllTests()));
    System.out.println(result);
    System.exit(0);
}