Search in sources :

Example 1 with SecurityHandler

use of org.apache.hadoop.oncrpc.security.SecurityHandler in project hadoop by apache.

the class TestWrites method testOverlappingWrites.

@Test
public void testOverlappingWrites() throws IOException, InterruptedException {
    NfsConfiguration config = new NfsConfiguration();
    MiniDFSCluster cluster = null;
    RpcProgramNfs3 nfsd;
    final int bufSize = 32;
    SecurityHandler securityHandler = Mockito.mock(SecurityHandler.class);
    Mockito.when(securityHandler.getUser()).thenReturn(System.getProperty("user.name"));
    String currentUser = System.getProperty("user.name");
    config.set(DefaultImpersonationProvider.getTestProvider().getProxySuperuserGroupConfKey(currentUser), "*");
    config.set(DefaultImpersonationProvider.getTestProvider().getProxySuperuserIpConfKey(currentUser), "*");
    ProxyUsers.refreshSuperUserGroupsConfiguration(config);
    // Use emphral port in case tests are running in parallel
    config.setInt("nfs3.mountd.port", 0);
    config.setInt("nfs3.server.port", 0);
    try {
        cluster = new MiniDFSCluster.Builder(config).numDataNodes(1).build();
        cluster.waitActive();
        Nfs3 nfs3 = new Nfs3(config);
        nfs3.startServiceInternal(false);
        nfsd = (RpcProgramNfs3) nfs3.getRpcProgram();
        DFSClient dfsClient = new DFSClient(DFSUtilClient.getNNAddress(config), config);
        HdfsFileStatus status = dfsClient.getFileInfo("/");
        FileHandle rootHandle = new FileHandle(status.getFileId());
        CREATE3Request createReq = new CREATE3Request(rootHandle, "overlapping-writes" + System.currentTimeMillis(), Nfs3Constant.CREATE_UNCHECKED, new SetAttr3(), 0);
        XDR createXdr = new XDR();
        createReq.serialize(createXdr);
        CREATE3Response createRsp = nfsd.create(createXdr.asReadOnlyWrap(), securityHandler, new InetSocketAddress("localhost", 1234));
        FileHandle handle = createRsp.getObjHandle();
        byte[] buffer = new byte[bufSize];
        for (int i = 0; i < bufSize; i++) {
            buffer[i] = (byte) i;
        }
        int[][] ranges = new int[][] { { 0, 10 }, { 5, 7 }, { 5, 5 }, { 10, 6 }, { 18, 6 }, { 20, 6 }, { 28, 4 }, { 16, 2 }, { 25, 4 } };
        for (int i = 0; i < ranges.length; i++) {
            int[] x = ranges[i];
            byte[] tbuffer = new byte[x[1]];
            for (int j = 0; j < x[1]; j++) {
                tbuffer[j] = buffer[x[0] + j];
            }
            WRITE3Request writeReq = new WRITE3Request(handle, (long) x[0], x[1], WriteStableHow.UNSTABLE, ByteBuffer.wrap(tbuffer));
            XDR writeXdr = new XDR();
            writeReq.serialize(writeXdr);
            nfsd.write(writeXdr.asReadOnlyWrap(), null, 1, securityHandler, new InetSocketAddress("localhost", 1234));
        }
        waitWrite(nfsd, handle, 60000);
        READ3Request readReq = new READ3Request(handle, 0, bufSize);
        XDR readXdr = new XDR();
        readReq.serialize(readXdr);
        READ3Response readRsp = nfsd.read(readXdr.asReadOnlyWrap(), securityHandler, new InetSocketAddress("localhost", config.getInt(NfsConfigKeys.DFS_NFS_SERVER_PORT_KEY, NfsConfigKeys.DFS_NFS_SERVER_PORT_DEFAULT)));
        assertTrue(Arrays.equals(buffer, readRsp.getData().array()));
    } finally {
        if (cluster != null) {
            cluster.shutdown();
        }
    }
}
Also used : DFSClient(org.apache.hadoop.hdfs.DFSClient) SecurityHandler(org.apache.hadoop.oncrpc.security.SecurityHandler) MiniDFSCluster(org.apache.hadoop.hdfs.MiniDFSCluster) FileHandle(org.apache.hadoop.nfs.nfs3.FileHandle) InetSocketAddress(java.net.InetSocketAddress) XDR(org.apache.hadoop.oncrpc.XDR) WRITE3Request(org.apache.hadoop.nfs.nfs3.request.WRITE3Request) NfsConfiguration(org.apache.hadoop.hdfs.nfs.conf.NfsConfiguration) SetAttr3(org.apache.hadoop.nfs.nfs3.request.SetAttr3) READ3Request(org.apache.hadoop.nfs.nfs3.request.READ3Request) CREATE3Response(org.apache.hadoop.nfs.nfs3.response.CREATE3Response) HdfsFileStatus(org.apache.hadoop.hdfs.protocol.HdfsFileStatus) CREATE3Request(org.apache.hadoop.nfs.nfs3.request.CREATE3Request) READ3Response(org.apache.hadoop.nfs.nfs3.response.READ3Response) Test(org.junit.Test)

Example 2 with SecurityHandler

use of org.apache.hadoop.oncrpc.security.SecurityHandler in project hadoop by apache.

the class TestWrites method testOOOWrites.

@Test
public void testOOOWrites() throws IOException, InterruptedException {
    NfsConfiguration config = new NfsConfiguration();
    MiniDFSCluster cluster = null;
    RpcProgramNfs3 nfsd;
    final int bufSize = 32;
    final int numOOO = 3;
    SecurityHandler securityHandler = Mockito.mock(SecurityHandler.class);
    Mockito.when(securityHandler.getUser()).thenReturn(System.getProperty("user.name"));
    String currentUser = System.getProperty("user.name");
    config.set(DefaultImpersonationProvider.getTestProvider().getProxySuperuserGroupConfKey(currentUser), "*");
    config.set(DefaultImpersonationProvider.getTestProvider().getProxySuperuserIpConfKey(currentUser), "*");
    ProxyUsers.refreshSuperUserGroupsConfiguration(config);
    // Use emphral port in case tests are running in parallel
    config.setInt("nfs3.mountd.port", 0);
    config.setInt("nfs3.server.port", 0);
    try {
        cluster = new MiniDFSCluster.Builder(config).numDataNodes(1).build();
        cluster.waitActive();
        Nfs3 nfs3 = new Nfs3(config);
        nfs3.startServiceInternal(false);
        nfsd = (RpcProgramNfs3) nfs3.getRpcProgram();
        DFSClient dfsClient = new DFSClient(DFSUtilClient.getNNAddress(config), config);
        HdfsFileStatus status = dfsClient.getFileInfo("/");
        FileHandle rootHandle = new FileHandle(status.getFileId());
        CREATE3Request createReq = new CREATE3Request(rootHandle, "out-of-order-write" + System.currentTimeMillis(), Nfs3Constant.CREATE_UNCHECKED, new SetAttr3(), 0);
        XDR createXdr = new XDR();
        createReq.serialize(createXdr);
        CREATE3Response createRsp = nfsd.create(createXdr.asReadOnlyWrap(), securityHandler, new InetSocketAddress("localhost", 1234));
        FileHandle handle = createRsp.getObjHandle();
        byte[][] oooBuf = new byte[numOOO][bufSize];
        for (int i = 0; i < numOOO; i++) {
            Arrays.fill(oooBuf[i], (byte) i);
        }
        for (int i = 0; i < numOOO; i++) {
            final long offset = (numOOO - 1 - i) * bufSize;
            WRITE3Request writeReq = new WRITE3Request(handle, offset, bufSize, WriteStableHow.UNSTABLE, ByteBuffer.wrap(oooBuf[i]));
            XDR writeXdr = new XDR();
            writeReq.serialize(writeXdr);
            nfsd.write(writeXdr.asReadOnlyWrap(), null, 1, securityHandler, new InetSocketAddress("localhost", 1234));
        }
        waitWrite(nfsd, handle, 60000);
        READ3Request readReq = new READ3Request(handle, bufSize, bufSize);
        XDR readXdr = new XDR();
        readReq.serialize(readXdr);
        READ3Response readRsp = nfsd.read(readXdr.asReadOnlyWrap(), securityHandler, new InetSocketAddress("localhost", config.getInt(NfsConfigKeys.DFS_NFS_SERVER_PORT_KEY, NfsConfigKeys.DFS_NFS_SERVER_PORT_DEFAULT)));
        assertTrue(Arrays.equals(oooBuf[1], readRsp.getData().array()));
    } finally {
        if (cluster != null) {
            cluster.shutdown();
        }
    }
}
Also used : DFSClient(org.apache.hadoop.hdfs.DFSClient) SecurityHandler(org.apache.hadoop.oncrpc.security.SecurityHandler) MiniDFSCluster(org.apache.hadoop.hdfs.MiniDFSCluster) FileHandle(org.apache.hadoop.nfs.nfs3.FileHandle) InetSocketAddress(java.net.InetSocketAddress) XDR(org.apache.hadoop.oncrpc.XDR) WRITE3Request(org.apache.hadoop.nfs.nfs3.request.WRITE3Request) NfsConfiguration(org.apache.hadoop.hdfs.nfs.conf.NfsConfiguration) SetAttr3(org.apache.hadoop.nfs.nfs3.request.SetAttr3) READ3Request(org.apache.hadoop.nfs.nfs3.request.READ3Request) CREATE3Response(org.apache.hadoop.nfs.nfs3.response.CREATE3Response) HdfsFileStatus(org.apache.hadoop.hdfs.protocol.HdfsFileStatus) CREATE3Request(org.apache.hadoop.nfs.nfs3.request.CREATE3Request) READ3Response(org.apache.hadoop.nfs.nfs3.response.READ3Response) Test(org.junit.Test)

Example 3 with SecurityHandler

use of org.apache.hadoop.oncrpc.security.SecurityHandler in project hadoop by apache.

the class RpcProgramNfs3 method commit.

@Override
public COMMIT3Response commit(XDR xdr, RpcInfo info) {
    SecurityHandler securityHandler = getSecurityHandler(info);
    RpcCall rpcCall = (RpcCall) info.header();
    int xid = rpcCall.getXid();
    SocketAddress remoteAddress = info.remoteAddress();
    return commit(xdr, info.channel(), xid, securityHandler, remoteAddress);
}
Also used : SecurityHandler(org.apache.hadoop.oncrpc.security.SecurityHandler) SysSecurityHandler(org.apache.hadoop.oncrpc.security.SysSecurityHandler) RpcCall(org.apache.hadoop.oncrpc.RpcCall) SocketAddress(java.net.SocketAddress) InetSocketAddress(java.net.InetSocketAddress)

Example 4 with SecurityHandler

use of org.apache.hadoop.oncrpc.security.SecurityHandler in project hadoop by apache.

the class RpcProgramNfs3 method write.

@Override
public WRITE3Response write(XDR xdr, RpcInfo info) {
    SecurityHandler securityHandler = getSecurityHandler(info);
    RpcCall rpcCall = (RpcCall) info.header();
    int xid = rpcCall.getXid();
    SocketAddress remoteAddress = info.remoteAddress();
    return write(xdr, info.channel(), xid, securityHandler, remoteAddress);
}
Also used : SecurityHandler(org.apache.hadoop.oncrpc.security.SecurityHandler) SysSecurityHandler(org.apache.hadoop.oncrpc.security.SysSecurityHandler) RpcCall(org.apache.hadoop.oncrpc.RpcCall) SocketAddress(java.net.SocketAddress) InetSocketAddress(java.net.InetSocketAddress)

Example 5 with SecurityHandler

use of org.apache.hadoop.oncrpc.security.SecurityHandler in project hadoop by apache.

the class TestWrites method testWriteStableHow.

@Test
public void testWriteStableHow() throws IOException, InterruptedException {
    NfsConfiguration config = new NfsConfiguration();
    DFSClient client = null;
    MiniDFSCluster cluster = null;
    RpcProgramNfs3 nfsd;
    SecurityHandler securityHandler = Mockito.mock(SecurityHandler.class);
    Mockito.when(securityHandler.getUser()).thenReturn(System.getProperty("user.name"));
    String currentUser = System.getProperty("user.name");
    config.set(DefaultImpersonationProvider.getTestProvider().getProxySuperuserGroupConfKey(currentUser), "*");
    config.set(DefaultImpersonationProvider.getTestProvider().getProxySuperuserIpConfKey(currentUser), "*");
    ProxyUsers.refreshSuperUserGroupsConfiguration(config);
    try {
        cluster = new MiniDFSCluster.Builder(config).numDataNodes(1).build();
        cluster.waitActive();
        client = new DFSClient(DFSUtilClient.getNNAddress(config), config);
        // Use emphral port in case tests are running in parallel
        config.setInt("nfs3.mountd.port", 0);
        config.setInt("nfs3.server.port", 0);
        // Start nfs
        Nfs3 nfs3 = new Nfs3(config);
        nfs3.startServiceInternal(false);
        nfsd = (RpcProgramNfs3) nfs3.getRpcProgram();
        HdfsFileStatus status = client.getFileInfo("/");
        FileHandle rootHandle = new FileHandle(status.getFileId());
        // Create file1
        CREATE3Request createReq = new CREATE3Request(rootHandle, "file1", Nfs3Constant.CREATE_UNCHECKED, new SetAttr3(), 0);
        XDR createXdr = new XDR();
        createReq.serialize(createXdr);
        CREATE3Response createRsp = nfsd.create(createXdr.asReadOnlyWrap(), securityHandler, new InetSocketAddress("localhost", 1234));
        FileHandle handle = createRsp.getObjHandle();
        // Test DATA_SYNC
        byte[] buffer = new byte[10];
        for (int i = 0; i < 10; i++) {
            buffer[i] = (byte) i;
        }
        WRITE3Request writeReq = new WRITE3Request(handle, 0, 10, WriteStableHow.DATA_SYNC, ByteBuffer.wrap(buffer));
        XDR writeXdr = new XDR();
        writeReq.serialize(writeXdr);
        nfsd.write(writeXdr.asReadOnlyWrap(), null, 1, securityHandler, new InetSocketAddress("localhost", 1234));
        waitWrite(nfsd, handle, 60000);
        // Readback
        READ3Request readReq = new READ3Request(handle, 0, 10);
        XDR readXdr = new XDR();
        readReq.serialize(readXdr);
        READ3Response readRsp = nfsd.read(readXdr.asReadOnlyWrap(), securityHandler, new InetSocketAddress("localhost", 1234));
        assertTrue(Arrays.equals(buffer, readRsp.getData().array()));
        // Test FILE_SYNC
        // Create file2
        CREATE3Request createReq2 = new CREATE3Request(rootHandle, "file2", Nfs3Constant.CREATE_UNCHECKED, new SetAttr3(), 0);
        XDR createXdr2 = new XDR();
        createReq2.serialize(createXdr2);
        CREATE3Response createRsp2 = nfsd.create(createXdr2.asReadOnlyWrap(), securityHandler, new InetSocketAddress("localhost", 1234));
        FileHandle handle2 = createRsp2.getObjHandle();
        WRITE3Request writeReq2 = new WRITE3Request(handle2, 0, 10, WriteStableHow.FILE_SYNC, ByteBuffer.wrap(buffer));
        XDR writeXdr2 = new XDR();
        writeReq2.serialize(writeXdr2);
        nfsd.write(writeXdr2.asReadOnlyWrap(), null, 1, securityHandler, new InetSocketAddress("localhost", 1234));
        waitWrite(nfsd, handle2, 60000);
        // Readback
        READ3Request readReq2 = new READ3Request(handle2, 0, 10);
        XDR readXdr2 = new XDR();
        readReq2.serialize(readXdr2);
        READ3Response readRsp2 = nfsd.read(readXdr2.asReadOnlyWrap(), securityHandler, new InetSocketAddress("localhost", 1234));
        assertTrue(Arrays.equals(buffer, readRsp2.getData().array()));
        // FILE_SYNC should sync the file size
        status = client.getFileInfo("/file2");
        assertTrue(status.getLen() == 10);
    } finally {
        if (cluster != null) {
            cluster.shutdown();
        }
    }
}
Also used : DFSClient(org.apache.hadoop.hdfs.DFSClient) SecurityHandler(org.apache.hadoop.oncrpc.security.SecurityHandler) MiniDFSCluster(org.apache.hadoop.hdfs.MiniDFSCluster) FileHandle(org.apache.hadoop.nfs.nfs3.FileHandle) InetSocketAddress(java.net.InetSocketAddress) XDR(org.apache.hadoop.oncrpc.XDR) WRITE3Request(org.apache.hadoop.nfs.nfs3.request.WRITE3Request) NfsConfiguration(org.apache.hadoop.hdfs.nfs.conf.NfsConfiguration) SetAttr3(org.apache.hadoop.nfs.nfs3.request.SetAttr3) READ3Request(org.apache.hadoop.nfs.nfs3.request.READ3Request) CREATE3Response(org.apache.hadoop.nfs.nfs3.response.CREATE3Response) HdfsFileStatus(org.apache.hadoop.hdfs.protocol.HdfsFileStatus) CREATE3Request(org.apache.hadoop.nfs.nfs3.request.CREATE3Request) READ3Response(org.apache.hadoop.nfs.nfs3.response.READ3Response) Test(org.junit.Test)

Aggregations

SecurityHandler (org.apache.hadoop.oncrpc.security.SecurityHandler)7 InetSocketAddress (java.net.InetSocketAddress)5 MiniDFSCluster (org.apache.hadoop.hdfs.MiniDFSCluster)5 DFSClient (org.apache.hadoop.hdfs.DFSClient)3 NfsConfiguration (org.apache.hadoop.hdfs.nfs.conf.NfsConfiguration)3 HdfsFileStatus (org.apache.hadoop.hdfs.protocol.HdfsFileStatus)3 FileHandle (org.apache.hadoop.nfs.nfs3.FileHandle)3 CREATE3Request (org.apache.hadoop.nfs.nfs3.request.CREATE3Request)3 READ3Request (org.apache.hadoop.nfs.nfs3.request.READ3Request)3 SetAttr3 (org.apache.hadoop.nfs.nfs3.request.SetAttr3)3 WRITE3Request (org.apache.hadoop.nfs.nfs3.request.WRITE3Request)3 CREATE3Response (org.apache.hadoop.nfs.nfs3.response.CREATE3Response)3 READ3Response (org.apache.hadoop.nfs.nfs3.response.READ3Response)3 XDR (org.apache.hadoop.oncrpc.XDR)3 Test (org.junit.Test)3 SocketAddress (java.net.SocketAddress)2 RpcCall (org.apache.hadoop.oncrpc.RpcCall)2 SysSecurityHandler (org.apache.hadoop.oncrpc.security.SysSecurityHandler)2 BeforeClass (org.junit.BeforeClass)2 File (java.io.File)1