Examples with StreamTokenizer java.io.StreamTokenizer used on opensource projects

Example 36 with StreamTokenizer

use of java.io.StreamTokenizer in project j2objc by google.

the class OldStreamTokenizerTest method test_nextToken.

public void test_nextToken() throws IOException {
    st = new StreamTokenizer(new Support_StringReader("\n \r\n#"));
    // make \n ordinary
    st.ordinaryChar('\n');
    st.eolIsSignificant(true);
    assertTrue("Wrong token 2,1", st.nextToken() == '\n');
    assertTrue("Wrong token 2,2", st.nextToken() == '\n');
    assertEquals("Wrong token 2,3", '#', st.nextToken());
    Support_ASimpleInputStream sis = new Support_ASimpleInputStream();
    sis.throwExceptionOnNextUse = true;
    st = new StreamTokenizer(sis);
    try {
        st.nextToken();
        fail("IOException expected.");
    } catch (IOException e) {
    // Expected.
    }
}

Example 37 with StreamTokenizer

use of java.io.StreamTokenizer in project j2objc by google.

the class StreamTokenizerTest method testLowerCase.

public void testLowerCase() throws Exception {
    Locale.setDefault(Locale.US);
    StreamTokenizer st = new StreamTokenizer(new StringReader("aIb aIb"));
    st.lowerCaseMode(true);
    st.nextToken();
    assertEquals("aib", st.sval);
    Locale oldDefault = Locale.getDefault();
    try {
        Locale.setDefault(new Locale("tr", "TR"));
        st.nextToken();
        assertEquals("aıb", st.sval);
    } finally {
        Locale.setDefault(oldDefault);
    }
}

Example 38 with StreamTokenizer

use of java.io.StreamTokenizer in project robovm by robovm.

the class OldAndroidStreamTokenizerTest method testStreamTokenizer.

public void testStreamTokenizer() throws Exception {
    String str = "Testing 12345 \n alpha \r\n omega";
    String strb = "-3.8 'BLIND mice' \r sEe /* how */ they run";
    StringReader aa = new StringReader(str);
    StringReader ba = new StringReader(strb);
    StreamTokenizer a = new StreamTokenizer(aa);
    StreamTokenizer b = new StreamTokenizer(ba);
    assertEquals(1, a.lineno());
    assertEquals(StreamTokenizer.TT_WORD, a.nextToken());
    assertEquals("Token[Testing], line 1", a.toString());
    assertEquals(StreamTokenizer.TT_NUMBER, a.nextToken());
    assertEquals("Token[n=12345.0], line 1", a.toString());
    assertEquals(StreamTokenizer.TT_WORD, a.nextToken());
    assertEquals("Token[alpha], line 2", a.toString());
    assertEquals(StreamTokenizer.TT_WORD, a.nextToken());
    assertEquals("Token[omega], line 3", a.toString());
    assertEquals(StreamTokenizer.TT_EOF, a.nextToken());
    assertEquals("Token[EOF], line 3", a.toString());
    b.commentChar('u');
    b.eolIsSignificant(true);
    b.lowerCaseMode(true);
    b.ordinaryChar('y');
    b.slashStarComments(true);
    assertEquals(StreamTokenizer.TT_NUMBER, b.nextToken());
    assertEquals(-3.8, b.nval);
    assertEquals("Token[n=-3.8], line 1", b.toString());
    // '
    assertEquals(39, b.nextToken());
    assertEquals("Token[BLIND mice], line 1", b.toString());
    // \n
    assertEquals(10, b.nextToken());
    assertEquals("Token[EOL], line 2", b.toString());
    assertEquals(StreamTokenizer.TT_WORD, b.nextToken());
    assertEquals("Token[see], line 2", b.toString());
    assertEquals(StreamTokenizer.TT_WORD, b.nextToken());
    assertEquals("Token[the], line 2", b.toString());
    // y
    assertEquals(121, b.nextToken());
    assertEquals("Token['y'], line 2", b.toString());
    assertEquals(StreamTokenizer.TT_WORD, b.nextToken());
    assertEquals("Token[r], line 2", b.toString());
    assertEquals(StreamTokenizer.TT_EOF, b.nextToken());
    assertEquals("Token[EOF], line 2", b.toString());
    // A harmony regression test
    byte[] data = new byte[] { (byte) '-' };
    StreamTokenizer tokenizer = new StreamTokenizer(new ByteArrayInputStream(data));
    tokenizer.nextToken();
    String result = tokenizer.toString();
    assertEquals("Token['-'], line 1", result);
    // another harmony regression test
    byte[] data2 = new byte[] { (byte) '"', (byte) 'H', (byte) 'e', (byte) 'l', (byte) 'l', (byte) 'o', (byte) '"' };
    StreamTokenizer tokenizer2 = new StreamTokenizer(new ByteArrayInputStream(data2));
    tokenizer2.nextToken();
    result = tokenizer2.toString();
    assertEquals("Token[Hello], line 1", result);
}

Example 39 with StreamTokenizer

use of java.io.StreamTokenizer in project platform_frameworks_base by android.

the class TypedProperties method initTokenizer.

/**
     * Instantiates a {@link java.io.StreamTokenizer} and sets its syntax tables
     * appropriately for the {@code TypedProperties} file format.
     *
     * @param r The {@code Reader} that the {@code StreamTokenizer} will read from
     * @return a newly-created and initialized {@code StreamTokenizer}
     */
static StreamTokenizer initTokenizer(Reader r) {
    StreamTokenizer st = new StreamTokenizer(r);
    // Treat everything we don't specify as "ordinary".
    st.resetSyntax();
    /* The only non-quoted-string words we'll be reading are:
         * - property names: [._$a-zA-Z0-9]
         * - type names: [a-zS]
         * - number literals: [-0-9.eExXA-Za-z]  ('x' for 0xNNN hex literals. "NaN", "Infinity")
         * - "true" or "false" (case insensitive): [a-zA-Z]
         */
    st.wordChars('0', '9');
    st.wordChars('A', 'Z');
    st.wordChars('a', 'z');
    st.wordChars('_', '_');
    st.wordChars('$', '$');
    st.wordChars('.', '.');
    st.wordChars('-', '-');
    st.wordChars('+', '+');
    // Single-character tokens
    st.ordinaryChar('=');
    // Other special characters
    st.whitespaceChars(' ', ' ');
    st.whitespaceChars('\t', '\t');
    st.whitespaceChars('\n', '\n');
    st.whitespaceChars('\r', '\r');
    st.quoteChar('"');
    // Java-style comments
    st.slashStarComments(true);
    st.slashSlashComments(true);
    return st;
}

Example 40 with StreamTokenizer

use of java.io.StreamTokenizer in project watson by totemo.

the class CalcCommand method makeTokenizer.

// concat
// --------------------------------------------------------------------------
/**
   * Create a StreamTokenizer that tokenises the specified command line
   * arguments.
   * 
   * @param args command line arguments string.
   * @return a new StreamTokenizer that tokenises the args.
   */
private static StreamTokenizer makeTokenizer(String args) {
    // Currently not supporting variables, but define words for future use.
    StreamTokenizer tokenizer = new StreamTokenizer(new StringReader(args));
    tokenizer.slashSlashComments(false);
    tokenizer.slashStarComments(false);
    tokenizer.wordChars('A', 'Z');
    tokenizer.wordChars('a', 'z');
    tokenizer.wordChars('_', '_');
    // These need to be defined as ordinary or they will be parsed as comment
    // introducers. >.<
    tokenizer.ordinaryChar('*');
    tokenizer.ordinaryChar('/');
    return tokenizer;
}