Examples with UnderFileSystemConfiguration alluxio.underfs.UnderFileSystemConfiguration used on opensource projects

Example 16 with UnderFileSystemConfiguration

use of alluxio.underfs.UnderFileSystemConfiguration in project alluxio by Alluxio.

the class S3AUnderFileSystemTest method getPermissionsNoMapping.

@Test
public void getPermissionsNoMapping() throws Exception {
    Map<PropertyKey, Object> conf = new HashMap<>();
    conf.put(PropertyKey.UNDERFS_S3_OWNER_ID_TO_USERNAME_MAPPING, "111=userid");
    try (Closeable c = new ConfigurationRule(conf, sConf).toResource()) {
        UnderFileSystemConfiguration ufsConf = UnderFileSystemConfiguration.defaults(sConf);
        mS3UnderFileSystem = new S3AUnderFileSystem(new AlluxioURI("s3a://" + BUCKET_NAME), mClient, BUCKET_NAME, mExecutor, mManager, UnderFileSystemConfiguration.defaults(sConf), false);
    }
    Mockito.when(mClient.getS3AccountOwner()).thenReturn(new Owner("0", "test"));
    Mockito.when(mClient.getBucketAcl(Mockito.anyString())).thenReturn(new AccessControlList());
    ObjectUnderFileSystem.ObjectPermissions permissions = mS3UnderFileSystem.getPermissions();
    Assert.assertEquals("test", permissions.getOwner());
    Assert.assertEquals("test", permissions.getGroup());
    Assert.assertEquals(0, permissions.getMode());
}

Example 17 with UnderFileSystemConfiguration

use of alluxio.underfs.UnderFileSystemConfiguration in project alluxio by Alluxio.

the class HdfsUnderFileSystemTest method prepareConfiguration.

/**
 * Tests the {@link HdfsUnderFileSystem#createConfiguration} method.
 *
 * Checks the hdfs implements class and alluxio underfs config setting
 */
@Test
public void prepareConfiguration() throws Exception {
    UnderFileSystemConfiguration ufsConf = UnderFileSystemConfiguration.defaults(ConfigurationTestUtils.defaults());
    org.apache.hadoop.conf.Configuration conf = HdfsUnderFileSystem.createConfiguration(ufsConf);
    Assert.assertEquals(ufsConf.get(PropertyKey.UNDERFS_HDFS_IMPL), conf.get("fs.hdfs.impl"));
    Assert.assertTrue(conf.getBoolean("fs.hdfs.impl.disable.cache", false));
}

Example 18 with UnderFileSystemConfiguration

use of alluxio.underfs.UnderFileSystemConfiguration in project alluxio by Alluxio.

the class KodoUnderFileSystem method creatInstance.

protected static KodoUnderFileSystem creatInstance(AlluxioURI uri, UnderFileSystemConfiguration conf) {
    String bucketName = UnderFileSystemUtils.getBucketName(uri);
    Preconditions.checkArgument(conf.isSet(PropertyKey.KODO_ACCESS_KEY), "Property %s is required to connect to Kodo", PropertyKey.KODO_ACCESS_KEY);
    Preconditions.checkArgument(conf.isSet(PropertyKey.KODO_SECRET_KEY), "Property %s is required to connect to Kodo", PropertyKey.KODO_SECRET_KEY);
    Preconditions.checkArgument(conf.isSet(PropertyKey.KODO_DOWNLOAD_HOST), "Property %s is required to connect to Kodo", PropertyKey.KODO_DOWNLOAD_HOST);
    Preconditions.checkArgument(conf.isSet(PropertyKey.KODO_ENDPOINT), "Property %s is required to connect to Kodo", PropertyKey.KODO_ENDPOINT);
    String accessKey = conf.getString(PropertyKey.KODO_ACCESS_KEY);
    String secretKey = conf.getString(PropertyKey.KODO_SECRET_KEY);
    String endPoint = conf.getString(PropertyKey.KODO_ENDPOINT);
    String souceHost = conf.getString(PropertyKey.KODO_DOWNLOAD_HOST);
    Auth auth = Auth.create(accessKey, secretKey);
    Configuration configuration = new Configuration();
    OkHttpClient.Builder okHttpBuilder = initializeKodoClientConfig(conf);
    OkHttpClient okHttpClient = okHttpBuilder.build();
    KodoClient kodoClient = new KodoClient(auth, bucketName, souceHost, endPoint, configuration, okHttpClient);
    return new KodoUnderFileSystem(uri, kodoClient, conf);
}

Example 19 with UnderFileSystemConfiguration

use of alluxio.underfs.UnderFileSystemConfiguration in project alluxio by Alluxio.

the class AdlUnderFileSystem method createConfiguration.

/**
 * Prepares the configuration for this Adl as an HDFS configuration.
 *
 * @param conf the configuration for this UFS
 * @return the created configuration
 */
public static Configuration createConfiguration(UnderFileSystemConfiguration conf) {
    Configuration adlConf = HdfsUnderFileSystem.createConfiguration(conf);
    for (Map.Entry<String, Object> entry : conf.toMap().entrySet()) {
        String key = entry.getKey();
        Object value = entry.getValue();
        if (PropertyKey.Template.UNDERFS_AZURE_CLIENT_ID.matches(key)) {
            adlConf.set(key, (String) value);
        }
        if (PropertyKey.Template.UNDERFS_AZURE_CLIENT_SECRET.matches(key)) {
            adlConf.set(key, (String) value);
        }
        if (PropertyKey.Template.UNDERFS_AZURE_REFRESH_URL.matches(key)) {
            adlConf.set(key, (String) value);
        }
    }
    LOG.info(adlConf.toString());
    adlConf.set("fs.adl.oauth2.access.token.provider.type", "ClientCredential");
    return adlConf;
}

Example 20 with UnderFileSystemConfiguration

use of alluxio.underfs.UnderFileSystemConfiguration in project alluxio by Alluxio.

the class UfsConfigurationJournalTest method testOptionsPersisted.

@Test
public void testOptionsPersisted() throws Exception {
    // Set ufs specific options and other mount flags
    AlluxioURI mountPoint = new AlluxioURI("/mnt");
    ImmutableMap<String, String> options = ImmutableMap.of("k1", "v1", "k2", "v2");
    mFs.mount(mountPoint, new AlluxioURI(LOCAL_UFS_PATH), MountPOptions.newBuilder().putAllProperties(options).setReadOnly(true).setShared(true).build());
    // Get mount id
    MountTable mountTable = Whitebox.getInternalState(mLocalAlluxioClusterResource.get().getLocalAlluxioMaster().getMasterProcess().getMaster(FileSystemMaster.class), "mMountTable");
    long mountId = mountTable.resolve(mountPoint).getMountId();
    // Restart masters
    mLocalAlluxioClusterResource.get().restartMasters();
    // Checks all options and flags are persisted after restart
    UfsManager ufsManager = Whitebox.getInternalState(mLocalAlluxioClusterResource.get().getLocalAlluxioMaster().getMasterProcess().getMaster(FileSystemMaster.class), "mUfsManager");
    try (CloseableResource<UnderFileSystem> resource = ufsManager.get(mountId).acquireUfsResource()) {
        UnderFileSystemConfiguration ufsConf = Whitebox.getInternalState(resource.get(), "mConf");
        assertEquals(ufsConf.getMountSpecificConf().size(), options.size());
        for (Map.Entry<String, String> entry : options.entrySet()) {
            assertEquals(entry.getValue(), ufsConf.getMountSpecificConf().get(entry.getKey()));
        }
        assertTrue(ufsConf.isReadOnly());
        assertTrue(ufsConf.isShared());
    }
}