Examples with StringValue org.apache.flink.types.StringValue used on opensource projects

Example 61 with StringValue

use of org.apache.flink.types.StringValue in project flink by apache.

the class StringValueUtilsTest method testTokenizerOnStringWithoutNexToken.

@Test
public void testTokenizerOnStringWithoutNexToken() {
    StringValue testString = new StringValue("test");
    StringValueUtils.WhitespaceTokenizer tokenizer = new WhitespaceTokenizer();
    tokenizer.setStringToTokenize(testString);
    // first token
    tokenizer.next(testString);
    // next token is not exist
    assertFalse(tokenizer.next(testString));
}

Example 62 with StringValue

use of org.apache.flink.types.StringValue in project flink by apache.

the class VarLengthStringParserTest method testParseValidQuotedStrings.

@Test
public void testParseValidQuotedStrings() {
    this.parser = new StringValueParser();
    this.parser.enableQuotedStringParsing((byte) '"');
    // check valid strings with out whitespaces and trailing delimiter
    byte[] recBytes = "\"abcdefgh\"|\"i\"|\"jklmno\"|".getBytes(ConfigConstants.DEFAULT_CHARSET);
    StringValue s = new StringValue();
    int startPos = 0;
    startPos = parser.parseField(recBytes, startPos, recBytes.length, new byte[] { '|' }, s);
    assertTrue(startPos == 11);
    assertTrue(s.getValue().equals("abcdefgh"));
    startPos = parser.parseField(recBytes, startPos, recBytes.length, new byte[] { '|' }, s);
    assertTrue(startPos == 15);
    assertTrue(s.getValue().equals("i"));
    startPos = parser.parseField(recBytes, startPos, recBytes.length, new byte[] { '|' }, s);
    assertTrue(startPos == 24);
    assertTrue(s.getValue().equals("jklmno"));
    // check single field not terminated
    recBytes = "\"abcde\"".getBytes(ConfigConstants.DEFAULT_CHARSET);
    startPos = 0;
    startPos = parser.parseField(recBytes, startPos, recBytes.length, new byte[] { '|' }, s);
    assertTrue(startPos == 7);
    assertTrue(s.getValue().equals("abcde"));
    // check last field not terminated
    recBytes = "\"abcde\"|\"fg\"".getBytes(ConfigConstants.DEFAULT_CHARSET);
    startPos = 0;
    startPos = parser.parseField(recBytes, startPos, recBytes.length, new byte[] { '|' }, s);
    assertTrue(startPos == 8);
    assertTrue(s.getValue().equals("abcde"));
    startPos = parser.parseField(recBytes, startPos, recBytes.length, new byte[] { '|' }, s);
    assertTrue(startPos == 12);
    assertTrue(s.getValue().equals("fg"));
    // check delimiter in quotes
    recBytes = "\"abcde|fg\"|\"hij|kl|mn|op\"|".getBytes(ConfigConstants.DEFAULT_CHARSET);
    startPos = 0;
    startPos = parser.parseField(recBytes, startPos, recBytes.length, new byte[] { '|' }, s);
    assertTrue(startPos == 11);
    assertTrue(s.getValue().equals("abcde|fg"));
    startPos = parser.parseField(recBytes, startPos, recBytes.length, new byte[] { '|' }, s);
    assertTrue(startPos == 26);
    assertTrue(s.getValue().equals("hij|kl|mn|op"));
    // check delimiter in quotes last field not terminated
    recBytes = "\"abcde|fg\"|\"hij|kl|mn|op\"".getBytes(ConfigConstants.DEFAULT_CHARSET);
    startPos = 0;
    startPos = parser.parseField(recBytes, startPos, recBytes.length, new byte[] { '|' }, s);
    assertTrue(startPos == 11);
    assertTrue(s.getValue().equals("abcde|fg"));
    startPos = parser.parseField(recBytes, startPos, recBytes.length, new byte[] { '|' }, s);
    assertTrue(startPos == 25);
    assertTrue(s.getValue().equals("hij|kl|mn|op"));
}

Example 63 with StringValue

use of org.apache.flink.types.StringValue in project flink by apache.

the class VarLengthStringParserTest method testParseInvalidQuotedStrings.

@Test
public void testParseInvalidQuotedStrings() {
    this.parser = new StringValueParser();
    this.parser.enableQuotedStringParsing((byte) '"');
    // check valid strings with out whitespaces and trailing delimiter
    byte[] recBytes = "\"abcdefgh\"-|\"jklmno  ".getBytes(ConfigConstants.DEFAULT_CHARSET);
    StringValue s = new StringValue();
    int startPos = 0;
    startPos = parser.parseField(recBytes, startPos, recBytes.length, new byte[] { '|' }, s);
    assertTrue(startPos < 0);
    startPos = 12;
    startPos = parser.parseField(recBytes, startPos, recBytes.length, new byte[] { '|' }, s);
    assertTrue(startPos < 0);
}

Example 64 with StringValue

use of org.apache.flink.types.StringValue in project flink by apache.

the class VarLengthStringParserTest method testParseValidMixedStrings.

@Test
public void testParseValidMixedStrings() {
    this.parser = new StringValueParser();
    this.parser.enableQuotedStringParsing((byte) '@');
    // check valid strings with out whitespaces and trailing delimiter
    byte[] recBytes = "@abcde|gh@|@i@|jklmnopq|@rs@|tuv".getBytes(ConfigConstants.DEFAULT_CHARSET);
    StringValue s = new StringValue();
    int startPos = 0;
    startPos = parser.parseField(recBytes, startPos, recBytes.length, new byte[] { '|' }, s);
    assertTrue(startPos == 11);
    assertTrue(s.getValue().equals("abcde|gh"));
    startPos = parser.parseField(recBytes, startPos, recBytes.length, new byte[] { '|' }, s);
    assertTrue(startPos == 15);
    assertTrue(s.getValue().equals("i"));
    startPos = parser.parseField(recBytes, startPos, recBytes.length, new byte[] { '|' }, s);
    assertTrue(startPos == 24);
    assertTrue(s.getValue().equals("jklmnopq"));
    startPos = parser.parseField(recBytes, startPos, recBytes.length, new byte[] { '|' }, s);
    assertTrue(startPos == 29);
    assertTrue(s.getValue().equals("rs"));
    startPos = parser.parseField(recBytes, startPos, recBytes.length, new byte[] { '|' }, s);
    assertTrue(startPos == 32);
    assertTrue(s.getValue().equals("tuv"));
}

Example 65 with StringValue

use of org.apache.flink.types.StringValue in project flink by apache.

the class VarLengthStringParserTest method testGetValue.

@Test
public void testGetValue() {
    Value v = parser.createValue();
    assertTrue(v instanceof StringValue);
}