Search in sources :

Example 66 with Configuration

use of org.apache.hadoop.conf.Configuration in project hadoop by apache.

the class TestFsShellReturnCode method testChownUserAndGroupValidity.

private void testChownUserAndGroupValidity(boolean enableWarning) {
    Configuration conf = new Configuration();
    conf.setBoolean(HADOOP_SHELL_MISSING_DEFAULT_FS_WARNING_KEY, enableWarning);
    FsCommand chown = new FakeChown();
    chown.setConf(conf);
    // The following are valid (no exception expected).
    chown.run("user", "/path");
    chown.run("user:group", "/path");
    chown.run(":group", "/path");
    // The following are valid only on Windows.
    assertValidArgumentsOnWindows(chown, "User With Spaces", "/path");
    assertValidArgumentsOnWindows(chown, "User With Spaces:group", "/path");
    assertValidArgumentsOnWindows(chown, "User With Spaces:Group With Spaces", "/path");
    assertValidArgumentsOnWindows(chown, "user:Group With Spaces", "/path");
    assertValidArgumentsOnWindows(chown, ":Group With Spaces", "/path");
    // The following are invalid (exception expected).
    assertIllegalArguments(chown, "us!er", "/path");
    assertIllegalArguments(chown, "us^er", "/path");
    assertIllegalArguments(chown, "user:gr#oup", "/path");
    assertIllegalArguments(chown, "user:gr%oup", "/path");
    assertIllegalArguments(chown, ":gr#oup", "/path");
    assertIllegalArguments(chown, ":gr%oup", "/path");
}
Also used : Configuration(org.apache.hadoop.conf.Configuration) FsCommand(org.apache.hadoop.fs.shell.FsCommand)

Example 67 with Configuration

use of org.apache.hadoop.conf.Configuration in project hadoop by apache.

the class TestFileSystemCaching method testCacheDisabled.

@Test
public void testCacheDisabled() throws Exception {
    Configuration conf = new Configuration();
    conf.set("fs.uncachedfile.impl", FileSystem.getFileSystemClass("file", null).getName());
    conf.setBoolean("fs.uncachedfile.impl.disable.cache", true);
    FileSystem fs1 = FileSystem.get(new URI("uncachedfile://a"), conf);
    FileSystem fs2 = FileSystem.get(new URI("uncachedfile://a"), conf);
    assertNotSame(fs1, fs2);
}
Also used : Configuration(org.apache.hadoop.conf.Configuration) FileSystem(org.apache.hadoop.fs.FileSystem) URI(java.net.URI) Test(org.junit.Test)

Example 68 with Configuration

use of org.apache.hadoop.conf.Configuration in project hadoop by apache.

the class TestFileSystemCaching method testCacheEnabled.

@Test
public void testCacheEnabled() throws Exception {
    Configuration conf = new Configuration();
    conf.set("fs.cachedfile.impl", FileSystem.getFileSystemClass("file", null).getName());
    FileSystem fs1 = FileSystem.get(new URI("cachedfile://a"), conf);
    FileSystem fs2 = FileSystem.get(new URI("cachedfile://a"), conf);
    assertSame(fs1, fs2);
}
Also used : Configuration(org.apache.hadoop.conf.Configuration) FileSystem(org.apache.hadoop.fs.FileSystem) URI(java.net.URI) Test(org.junit.Test)

Example 69 with Configuration

use of org.apache.hadoop.conf.Configuration in project hadoop by apache.

the class TestFileSystemCaching method testUserFS.

@Test
public void testUserFS() throws Exception {
    final Configuration conf = new Configuration();
    conf.set("fs.cachedfile.impl", FileSystem.getFileSystemClass("file", null).getName());
    FileSystem fsU1 = FileSystem.get(new URI("cachedfile://a"), conf, "bar");
    FileSystem fsU2 = FileSystem.get(new URI("cachedfile://a"), conf, "foo");
    assertNotSame(fsU1, fsU2);
}
Also used : Configuration(org.apache.hadoop.conf.Configuration) FileSystem(org.apache.hadoop.fs.FileSystem) URI(java.net.URI) Test(org.junit.Test)

Example 70 with Configuration

use of org.apache.hadoop.conf.Configuration in project hadoop by apache.

the class TestFileSystemCaching method testCacheForUgi.

@SuppressWarnings("unchecked")
@Test
public <T extends TokenIdentifier> void testCacheForUgi() throws Exception {
    final Configuration conf = new Configuration();
    conf.set("fs.cachedfile.impl", FileSystem.getFileSystemClass("file", null).getName());
    UserGroupInformation ugiA = UserGroupInformation.createRemoteUser("foo");
    UserGroupInformation ugiB = UserGroupInformation.createRemoteUser("bar");
    FileSystem fsA = ugiA.doAs(new PrivilegedExceptionAction<FileSystem>() {

        @Override
        public FileSystem run() throws Exception {
            return FileSystem.get(new URI("cachedfile://a"), conf);
        }
    });
    FileSystem fsA1 = ugiA.doAs(new PrivilegedExceptionAction<FileSystem>() {

        @Override
        public FileSystem run() throws Exception {
            return FileSystem.get(new URI("cachedfile://a"), conf);
        }
    });
    //Since the UGIs are the same, we should have the same filesystem for both
    assertSame(fsA, fsA1);
    FileSystem fsB = ugiB.doAs(new PrivilegedExceptionAction<FileSystem>() {

        @Override
        public FileSystem run() throws Exception {
            return FileSystem.get(new URI("cachedfile://a"), conf);
        }
    });
    //Since the UGIs are different, we should end up with different filesystems
    //corresponding to the two UGIs
    assertNotSame(fsA, fsB);
    Token<T> t1 = mock(Token.class);
    UserGroupInformation ugiA2 = UserGroupInformation.createRemoteUser("foo");
    fsA = ugiA2.doAs(new PrivilegedExceptionAction<FileSystem>() {

        @Override
        public FileSystem run() throws Exception {
            return FileSystem.get(new URI("cachedfile://a"), conf);
        }
    });
    // Although the users in the UGI are same, they have different subjects
    // and so are different.
    assertNotSame(fsA, fsA1);
    ugiA.addToken(t1);
    fsA = ugiA.doAs(new PrivilegedExceptionAction<FileSystem>() {

        @Override
        public FileSystem run() throws Exception {
            return FileSystem.get(new URI("cachedfile://a"), conf);
        }
    });
    // Make sure that different UGI's with the same subject lead to the same
    // file system.
    assertSame(fsA, fsA1);
}
Also used : Configuration(org.apache.hadoop.conf.Configuration) FileSystem(org.apache.hadoop.fs.FileSystem) PrivilegedExceptionAction(java.security.PrivilegedExceptionAction) URI(java.net.URI) URISyntaxException(java.net.URISyntaxException) IOException(java.io.IOException) UserGroupInformation(org.apache.hadoop.security.UserGroupInformation) Test(org.junit.Test)

Aggregations

Configuration (org.apache.hadoop.conf.Configuration)5973 Test (org.junit.Test)3243 Path (org.apache.hadoop.fs.Path)1602 FileSystem (org.apache.hadoop.fs.FileSystem)903 IOException (java.io.IOException)850 YarnConfiguration (org.apache.hadoop.yarn.conf.YarnConfiguration)727 HdfsConfiguration (org.apache.hadoop.hdfs.HdfsConfiguration)517 MiniDFSCluster (org.apache.hadoop.hdfs.MiniDFSCluster)502 File (java.io.File)499 HBaseConfiguration (org.apache.hadoop.hbase.HBaseConfiguration)388 ArrayList (java.util.ArrayList)360 URI (java.net.URI)319 BeforeClass (org.junit.BeforeClass)275 Job (org.apache.hadoop.mapreduce.Job)272 Before (org.junit.Before)264 DistributedFileSystem (org.apache.hadoop.hdfs.DistributedFileSystem)219 FSDataOutputStream (org.apache.hadoop.fs.FSDataOutputStream)203 HashMap (java.util.HashMap)192 FileStatus (org.apache.hadoop.fs.FileStatus)190 Properties (java.util.Properties)187