Examples with FileSystemTestHelper org.apache.hadoop.fs.FileSystemTestHelper used on opensource projects

Example 1 with FileSystemTestHelper

use of org.apache.hadoop.fs.FileSystemTestHelper in project hadoop by apache.

the class TestEncryptionZones method setup.

@Before
public void setup() throws Exception {
    conf = new HdfsConfiguration();
    fsHelper = new FileSystemTestHelper();
    // Set up java key store
    String testRoot = fsHelper.getTestRootDir();
    testRootDir = new File(testRoot).getAbsoluteFile();
    conf.set(CommonConfigurationKeysPublic.HADOOP_SECURITY_KEY_PROVIDER_PATH, getKeyProviderURI());
    conf.setBoolean(DFSConfigKeys.DFS_NAMENODE_DELEGATION_TOKEN_ALWAYS_USE_KEY, true);
    // Lower the batch size for testing
    conf.setInt(DFSConfigKeys.DFS_NAMENODE_LIST_ENCRYPTION_ZONES_NUM_RESPONSES, 2);
    cluster = new MiniDFSCluster.Builder(conf).numDataNodes(1).build();
    cluster.waitActive();
    Logger.getLogger(EncryptionZoneManager.class).setLevel(Level.TRACE);
    fs = cluster.getFileSystem();
    fsWrapper = new FileSystemTestWrapper(fs);
    fcWrapper = new FileContextTestWrapper(FileContext.getFileContext(cluster.getURI(), conf));
    dfsAdmin = new HdfsAdmin(cluster.getURI(), conf);
    setProvider();
    // Create a test key
    DFSTestUtil.createKey(TEST_KEY, cluster, conf);
}

Example 2 with FileSystemTestHelper

use of org.apache.hadoop.fs.FileSystemTestHelper in project hadoop by apache.

the class TestNameNodeMetrics method testGenerateEDEKTime.

@Test
public void testGenerateEDEKTime() throws IOException, NoSuchAlgorithmException {
    //Create new MiniDFSCluster with EncryptionZone configurations
    Configuration conf = new HdfsConfiguration();
    FileSystemTestHelper fsHelper = new FileSystemTestHelper();
    // Set up java key store
    String testRoot = fsHelper.getTestRootDir();
    File testRootDir = new File(testRoot).getAbsoluteFile();
    conf.set(CommonConfigurationKeysPublic.HADOOP_SECURITY_KEY_PROVIDER_PATH, JavaKeyStoreProvider.SCHEME_NAME + "://file" + new Path(testRootDir.toString(), "test.jks").toUri());
    conf.setBoolean(DFSConfigKeys.DFS_NAMENODE_DELEGATION_TOKEN_ALWAYS_USE_KEY, true);
    conf.setInt(DFSConfigKeys.DFS_NAMENODE_LIST_ENCRYPTION_ZONES_NUM_RESPONSES, 2);
    try (MiniDFSCluster clusterEDEK = new MiniDFSCluster.Builder(conf).numDataNodes(1).build()) {
        DistributedFileSystem fsEDEK = clusterEDEK.getFileSystem();
        FileSystemTestWrapper fsWrapper = new FileSystemTestWrapper(fsEDEK);
        HdfsAdmin dfsAdmin = new HdfsAdmin(clusterEDEK.getURI(), conf);
        fsEDEK.getClient().setKeyProvider(clusterEDEK.getNameNode().getNamesystem().getProvider());
        String testKey = "test_key";
        DFSTestUtil.createKey(testKey, clusterEDEK, conf);
        final Path zoneParent = new Path("/zones");
        final Path zone1 = new Path(zoneParent, "zone1");
        fsWrapper.mkdir(zone1, FsPermission.getDirDefault(), true);
        dfsAdmin.createEncryptionZone(zone1, "test_key", EnumSet.of(CreateEncryptionZoneFlag.NO_TRASH));
        MetricsRecordBuilder rb = getMetrics(NN_METRICS);
        for (int i = 0; i < 3; i++) {
            Path filePath = new Path("/zones/zone1/testfile-" + i);
            DFSTestUtil.createFile(fsEDEK, filePath, 1024, (short) 3, 1L);
            assertQuantileGauges("GenerateEDEKTime1s", rb);
        }
    }
}

Example 3 with FileSystemTestHelper

use of org.apache.hadoop.fs.FileSystemTestHelper in project hadoop by apache.

the class TestTransferFsImage method testImageUploadTimeout.

/**
   * Test to verify the timeout of Image upload
   */
@Test(timeout = 10000)
public void testImageUploadTimeout() throws Exception {
    Configuration conf = new HdfsConfiguration();
    NNStorage mockStorage = Mockito.mock(NNStorage.class);
    HttpServer2 testServer = HttpServerFunctionalTest.createServer("hdfs");
    try {
        testServer.addServlet("ImageTransfer", ImageServlet.PATH_SPEC, TestImageTransferServlet.class);
        testServer.start();
        URL serverURL = HttpServerFunctionalTest.getServerURL(testServer);
        // set the timeout here, otherwise it will take default.
        TransferFsImage.timeout = 2000;
        File tmpDir = new File(new FileSystemTestHelper().getTestRootDir());
        tmpDir.mkdirs();
        File mockImageFile = File.createTempFile("image", "", tmpDir);
        FileOutputStream imageFile = new FileOutputStream(mockImageFile);
        imageFile.write("data".getBytes());
        imageFile.close();
        Mockito.when(mockStorage.findImageFile(Mockito.any(NameNodeFile.class), Mockito.anyLong())).thenReturn(mockImageFile);
        Mockito.when(mockStorage.toColonSeparatedString()).thenReturn("storage:info:string");
        try {
            TransferFsImage.uploadImageFromStorage(serverURL, conf, mockStorage, NameNodeFile.IMAGE, 1L);
            fail("TransferImage Should fail with timeout");
        } catch (SocketTimeoutException e) {
            assertEquals("Upload should timeout", "Read timed out", e.getMessage());
        }
    } finally {
        testServer.stop();
    }
}

Example 4 with FileSystemTestHelper

use of org.apache.hadoop.fs.FileSystemTestHelper in project hadoop by apache.

the class TestNestedEncryptionZones method setup.

@Before
public void setup() throws Exception {
    Configuration conf = new HdfsConfiguration();
    FileSystemTestHelper fsHelper = new FileSystemTestHelper();
    // Set up java key store
    String testRoot = fsHelper.getTestRootDir();
    testRootDir = new File(testRoot).getAbsoluteFile();
    conf.set(CommonConfigurationKeysPublic.HADOOP_SECURITY_KEY_PROVIDER_PATH, getKeyProviderURI());
    conf.setBoolean(DFSConfigKeys.DFS_NAMENODE_DELEGATION_TOKEN_ALWAYS_USE_KEY, true);
    // Lower the batch size for testing
    conf.setInt(DFSConfigKeys.DFS_NAMENODE_LIST_ENCRYPTION_ZONES_NUM_RESPONSES, 2);
    // enable trash for testing
    conf.setLong(DFSConfigKeys.FS_TRASH_INTERVAL_KEY, 1);
    cluster = new MiniDFSCluster.Builder(conf).numDataNodes(1).build();
    Logger.getLogger(EncryptionZoneManager.class).setLevel(Level.TRACE);
    fs = cluster.getFileSystem();
    setProvider();
    // Create test keys and EZs
    DFSTestUtil.createKey(TOP_EZ_KEY, cluster, conf);
    DFSTestUtil.createKey(NESTED_EZ_KEY, cluster, conf);
}

Example 5 with FileSystemTestHelper

use of org.apache.hadoop.fs.FileSystemTestHelper in project hadoop by apache.

the class TestCopyPreserveFlag method initialize.

@Before
public void initialize() throws Exception {
    conf = new Configuration(false);
    conf.set("fs.file.impl", LocalFileSystem.class.getName());
    fs = FileSystem.getLocal(conf);
    testDir = new FileSystemTestHelper().getTestRootPath(fs);
    // don't want scheme on the path, just an absolute path
    testDir = new Path(fs.makeQualified(testDir).toUri().getPath());
    FileSystem.setDefaultUri(conf, fs.getUri());
    fs.setWorkingDirectory(testDir);
    fs.mkdirs(DIR_FROM);
    fs.mkdirs(DIR_TO1);
    fs.createNewFile(FROM);
    FSDataOutputStream output = fs.create(FROM, true);
    for (int i = 0; i < 100; ++i) {
        output.writeInt(i);
        output.writeChar('\n');
    }
    output.close();
    fs.setTimes(FROM, MODIFICATION_TIME, ACCESS_TIME);
    fs.setPermission(FROM, PERMISSIONS);
    fs.setTimes(DIR_FROM, MODIFICATION_TIME, ACCESS_TIME);
    fs.setPermission(DIR_FROM, PERMISSIONS);
}