Search in sources :

Example 11 with BufferedWriter

use of java.io.BufferedWriter in project hadoop by apache.

the class TestConfiguration method testMultiplePropertySource.

public void testMultiplePropertySource() throws IOException {
    out = new BufferedWriter(new FileWriter(CONFIG));
    startConfig();
    appendProperty("test.foo", "bar", false, "a", "b", "c");
    endConfig();
    Path fileResource = new Path(CONFIG);
    conf.addResource(fileResource);
    String[] sources = conf.getPropertySources("test.foo");
    assertEquals(4, sources.length);
    assertEquals("a", sources[0]);
    assertEquals("b", sources[1]);
    assertEquals("c", sources[2]);
    assertEquals("Resource string returned for a file-loaded property" + " must be a proper absolute path", fileResource, new Path(sources[3]));
}
Also used : Path(org.apache.hadoop.fs.Path) FileWriter(java.io.FileWriter) BufferedWriter(java.io.BufferedWriter)

Example 12 with BufferedWriter

use of java.io.BufferedWriter in project hadoop by apache.

the class TestConfiguration method testVariableSubstitution.

public void testVariableSubstitution() throws IOException {
    // stubbing only environment dependent functions
    Configuration mock = Mockito.spy(conf);
    Mockito.when(mock.getProperty("user.name")).thenReturn("hadoop_user");
    Mockito.when(mock.getenv("FILE_NAME")).thenReturn("hello");
    out = new BufferedWriter(new FileWriter(CONFIG));
    startConfig();
    declareProperty("my.int", "${intvar}", "42");
    declareProperty("intvar", "42", "42");
    declareProperty("my.base", "/tmp/${user.name}", "/tmp/hadoop_user");
    declareProperty("my.file", "${env.FILE_NAME}", "hello");
    declareProperty("my.suffix", ".txt", ".txt");
    declareProperty("my.relfile", "${my.file}${my.suffix}", "hello.txt");
    declareProperty("my.fullfile", "${my.base}/${my.file}${my.suffix}", "/tmp/hadoop_user/hello.txt");
    // check that undefined variables are returned as-is
    declareProperty("my.failsexpand", "a${my.undefvar}b", "a${my.undefvar}b");
    // check that multiple variable references are resolved
    declareProperty("my.user.group", "${user.name} ${user.name}", "hadoop_user hadoop_user");
    endConfig();
    Path fileResource = new Path(CONFIG);
    mock.addResource(fileResource);
    for (Prop p : props) {
        System.out.println("p=" + p.name);
        String gotVal = mock.get(p.name);
        String gotRawVal = mock.getRaw(p.name);
        assertEq(p.val, gotRawVal);
        assertEq(p.expectEval, gotVal);
    }
    // check that expansion also occurs for getInt()
    assertTrue(mock.getInt("intvar", -1) == 42);
    assertTrue(mock.getInt("my.int", -1) == 42);
}
Also used : Path(org.apache.hadoop.fs.Path) FileWriter(java.io.FileWriter) BufferedWriter(java.io.BufferedWriter)

Example 13 with BufferedWriter

use of java.io.BufferedWriter in project hadoop by apache.

the class TestConfiguration method testToString.

public void testToString() throws IOException {
    out = new BufferedWriter(new FileWriter(CONFIG));
    startConfig();
    endConfig();
    Path fileResource = new Path(CONFIG);
    conf.addResource(fileResource);
    String expectedOutput = "Configuration: core-default.xml, core-site.xml, " + fileResource.toString();
    assertEquals(expectedOutput, conf.toString());
}
Also used : Path(org.apache.hadoop.fs.Path) FileWriter(java.io.FileWriter) BufferedWriter(java.io.BufferedWriter)

Example 14 with BufferedWriter

use of java.io.BufferedWriter in project hadoop by apache.

the class TestConfiguration method testDumpConfiguration.

public void testDumpConfiguration() throws IOException {
    StringWriter outWriter = new StringWriter();
    Configuration.dumpConfiguration(conf, outWriter);
    String jsonStr = outWriter.toString();
    ObjectMapper mapper = new ObjectMapper();
    JsonConfiguration jconf = mapper.readValue(jsonStr, JsonConfiguration.class);
    int defaultLength = jconf.getProperties().length;
    // add 3 keys to the existing configuration properties
    out = new BufferedWriter(new FileWriter(CONFIG));
    startConfig();
    appendProperty("test.key1", "value1");
    appendProperty("test.key2", "value2", true);
    appendProperty("test.key3", "value3");
    endConfig();
    Path fileResource = new Path(CONFIG);
    conf.addResource(fileResource);
    out.close();
    outWriter = new StringWriter();
    Configuration.dumpConfiguration(conf, outWriter);
    jsonStr = outWriter.toString();
    mapper = new ObjectMapper();
    jconf = mapper.readValue(jsonStr, JsonConfiguration.class);
    int length = jconf.getProperties().length;
    // check for consistency in the number of properties parsed in Json format.
    assertEquals(length, defaultLength + 3);
    //change few keys in another resource file
    out = new BufferedWriter(new FileWriter(CONFIG2));
    startConfig();
    appendProperty("test.key1", "newValue1");
    appendProperty("test.key2", "newValue2");
    endConfig();
    Path fileResource1 = new Path(CONFIG2);
    conf.addResource(fileResource1);
    out.close();
    outWriter = new StringWriter();
    Configuration.dumpConfiguration(conf, outWriter);
    jsonStr = outWriter.toString();
    mapper = new ObjectMapper();
    jconf = mapper.readValue(jsonStr, JsonConfiguration.class);
    // put the keys and their corresponding attributes into a hashmap for their 
    // efficient retrieval
    HashMap<String, JsonProperty> confDump = new HashMap<String, JsonProperty>();
    for (JsonProperty prop : jconf.getProperties()) {
        confDump.put(prop.getKey(), prop);
    }
    // check if the value and resource of test.key1 is changed
    assertEquals("newValue1", confDump.get("test.key1").getValue());
    assertEquals(false, confDump.get("test.key1").getIsFinal());
    assertEquals(fileResource1.toString(), confDump.get("test.key1").getResource());
    // check if final parameter test.key2 is not changed, since it is first 
    // loaded as final parameter
    assertEquals("value2", confDump.get("test.key2").getValue());
    assertEquals(true, confDump.get("test.key2").getIsFinal());
    assertEquals(fileResource.toString(), confDump.get("test.key2").getResource());
    // check for other keys which are not modified later
    assertEquals("value3", confDump.get("test.key3").getValue());
    assertEquals(false, confDump.get("test.key3").getIsFinal());
    assertEquals(fileResource.toString(), confDump.get("test.key3").getResource());
    // check for resource to be "Unknown" for keys which are loaded using 'set' 
    // and expansion of properties
    conf.set("test.key4", "value4");
    conf.set("test.key5", "value5");
    conf.set("test.key6", "${test.key5}");
    outWriter = new StringWriter();
    Configuration.dumpConfiguration(conf, outWriter);
    jsonStr = outWriter.toString();
    mapper = new ObjectMapper();
    jconf = mapper.readValue(jsonStr, JsonConfiguration.class);
    confDump = new HashMap<String, JsonProperty>();
    for (JsonProperty prop : jconf.getProperties()) {
        confDump.put(prop.getKey(), prop);
    }
    assertEquals("value5", confDump.get("test.key6").getValue());
    assertEquals("programmatically", confDump.get("test.key4").getResource());
    outWriter.close();
}
Also used : Path(org.apache.hadoop.fs.Path) StringWriter(java.io.StringWriter) HashMap(java.util.HashMap) FileWriter(java.io.FileWriter) ObjectMapper(com.fasterxml.jackson.databind.ObjectMapper) BufferedWriter(java.io.BufferedWriter)

Example 15 with BufferedWriter

use of java.io.BufferedWriter in project hadoop by apache.

the class TestConfiguration method testIncludes.

public void testIncludes() throws Exception {
    tearDown();
    System.out.println("XXX testIncludes");
    out = new BufferedWriter(new FileWriter(CONFIG2));
    startConfig();
    appendProperty("a", "b");
    appendProperty("c", "d");
    endConfig();
    out = new BufferedWriter(new FileWriter(CONFIG));
    startConfig();
    addInclude(CONFIG2);
    appendProperty("e", "f");
    appendProperty("g", "h");
    endConfig();
    // verify that the includes file contains all properties
    Path fileResource = new Path(CONFIG);
    conf.addResource(fileResource);
    assertEquals(conf.get("a"), "b");
    assertEquals(conf.get("c"), "d");
    assertEquals(conf.get("e"), "f");
    assertEquals(conf.get("g"), "h");
    tearDown();
}
Also used : Path(org.apache.hadoop.fs.Path) FileWriter(java.io.FileWriter) BufferedWriter(java.io.BufferedWriter)

Aggregations

BufferedWriter (java.io.BufferedWriter)1557 IOException (java.io.IOException)726 FileWriter (java.io.FileWriter)706 File (java.io.File)629 OutputStreamWriter (java.io.OutputStreamWriter)603 FileOutputStream (java.io.FileOutputStream)291 BufferedReader (java.io.BufferedReader)206 Writer (java.io.Writer)138 InputStreamReader (java.io.InputStreamReader)124 PrintWriter (java.io.PrintWriter)117 ArrayList (java.util.ArrayList)102 FileNotFoundException (java.io.FileNotFoundException)101 Test (org.junit.Test)99 Path (org.apache.hadoop.fs.Path)80 FileReader (java.io.FileReader)78 OutputStream (java.io.OutputStream)65 StringWriter (java.io.StringWriter)58 Path (java.nio.file.Path)58 Date (java.util.Date)57 FileInputStream (java.io.FileInputStream)54