Examples with Configuration org.apache.flink.configuration.Configuration used on opensource projects

Example 61 with Configuration

use of org.apache.flink.configuration.Configuration in project flink by apache.

the class DelimitedInputFormatSamplingTest method testCachedStatistics.

@Test
public void testCachedStatistics() {
    try {
        final String tempFile = TestFileUtils.createTempFile(TEST_DATA1);
        final Configuration conf = new Configuration();
        final TestDelimitedInputFormat format = new TestDelimitedInputFormat(CONFIG);
        format.setFilePath("test://" + tempFile);
        format.configure(conf);
        TestFileSystem.resetStreamOpenCounter();
        BaseStatistics stats = format.getStatistics(null);
        Assert.assertEquals("Wrong number of samples taken.", DEFAULT_NUM_SAMPLES, TestFileSystem.getNumtimeStreamOpened());
        final TestDelimitedInputFormat format2 = new TestDelimitedInputFormat(CONFIG);
        format2.setFilePath("test://" + tempFile);
        format2.configure(conf);
        TestFileSystem.resetStreamOpenCounter();
        BaseStatistics stats2 = format2.getStatistics(stats);
        Assert.assertTrue("Using cached statistics should cicumvent sampling.", 0 == TestFileSystem.getNumtimeStreamOpened());
        Assert.assertTrue("Using cached statistics should cicumvent sampling.", stats == stats2);
    } catch (Exception e) {
        e.printStackTrace();
        Assert.fail(e.getMessage());
    }
}

Example 62 with Configuration

use of org.apache.flink.configuration.Configuration in project flink by apache.

the class DelimitedInputFormatSamplingTest method testSamplingOneFile.

@Test
public void testSamplingOneFile() {
    try {
        final String tempFile = TestFileUtils.createTempFile(TEST_DATA1);
        final Configuration conf = new Configuration();
        final TestDelimitedInputFormat format = new TestDelimitedInputFormat(CONFIG);
        format.setFilePath(tempFile);
        format.configure(conf);
        BaseStatistics stats = format.getStatistics(null);
        final int numLines = TEST_DATA_1_LINES;
        final float avgWidth = ((float) TEST_DATA1.length()) / TEST_DATA_1_LINES;
        Assert.assertTrue("Wrong record count.", stats.getNumberOfRecords() < numLines + 1 & stats.getNumberOfRecords() > numLines - 1);
        Assert.assertTrue("Wrong avg record size.", stats.getAverageRecordWidth() < avgWidth + 1 & stats.getAverageRecordWidth() > avgWidth - 1);
    } catch (Exception e) {
        e.printStackTrace();
        Assert.fail(e.getMessage());
    }
}

Example 63 with Configuration

use of org.apache.flink.configuration.Configuration in project flink by apache.

the class DelimitedInputFormatTest method testReadCustomDelimiter.

@Test
public void testReadCustomDelimiter() throws IOException {
    final String myString = "my key|my val$$$my key2\n$$ctd.$$|my value2";
    final FileInputSplit split = createTempFile(myString);
    final Configuration parameters = new Configuration();
    format.setDelimiter("$$$");
    format.configure(parameters);
    format.open(split);
    String first = format.nextRecord(null);
    assertNotNull(first);
    assertEquals("my key|my val", first);
    String second = format.nextRecord(null);
    assertNotNull(second);
    assertEquals("my key2\n$$ctd.$$|my value2", second);
    assertNull(format.nextRecord(null));
    assertTrue(format.reachedEnd());
}

Example 64 with Configuration

use of org.apache.flink.configuration.Configuration in project flink by apache.

the class DelimitedInputFormatTest method testConfigure.

// --------------------------------------------------------------------------------------------
// --------------------------------------------------------------------------------------------
@Test
public void testConfigure() {
    Configuration cfg = new Configuration();
    cfg.setString("delimited-format.delimiter", "\n");
    format.configure(cfg);
    assertEquals("\n", new String(format.getDelimiter(), format.getCharset()));
    cfg.setString("delimited-format.delimiter", "&-&");
    format.configure(cfg);
    assertEquals("&-&", new String(format.getDelimiter(), format.getCharset()));
}

Example 65 with Configuration

use of org.apache.flink.configuration.Configuration in project flink by apache.

the class DelimitedInputFormatTest method testReadExactlyBufferSize.

@Test
public void testReadExactlyBufferSize() throws IOException {
    final String myString = "aaaaaaa\nbbbbbbb\nccccccc\nddddddd\n";
    final FileInputSplit split = createTempFile(myString);
    final Configuration parameters = new Configuration();
    format.setBufferSize((int) split.getLength());
    format.configure(parameters);
    format.open(split);
    String next;
    int count = 0;
    while ((next = format.nextRecord(null)) != null) {
        assertEquals(7, next.length());
        count++;
    }
    assertNull(format.nextRecord(null));
    assertTrue(format.reachedEnd());
    format.close();
    assertEquals(4, count);
}