Examples with StreamTokenizer java.io.StreamTokenizer used on opensource projects

Example 46 with StreamTokenizer

use of java.io.StreamTokenizer in project robovm by robovm.

the class OldStreamTokenizerTest method test_harmonyRegressionTest2.

public void test_harmonyRegressionTest2() {
    byte[] data = new byte[] { (byte) '"', (byte) 'H', (byte) 'e', (byte) 'l', (byte) 'l', (byte) 'o', (byte) '"' };
    StreamTokenizer tokenizer = new StreamTokenizer(new ByteArrayInputStream(data));
    try {
        tokenizer.nextToken();
    } catch (Exception e) {
        e.printStackTrace();
    }
    String result = tokenizer.toString();
    Assert.assertEquals("Token[Hello], line 1", result);
}

Example 47 with StreamTokenizer

use of java.io.StreamTokenizer in project robovm by robovm.

the class StreamTokenizerTest method testLowerCase.

public void testLowerCase() throws Exception {
    Locale.setDefault(Locale.US);
    StreamTokenizer st = new StreamTokenizer(new StringReader("aIb aIb"));
    st.lowerCaseMode(true);
    st.nextToken();
    assertEquals("aib", st.sval);
    Locale.setDefault(new Locale("tr", "TR"));
    st.nextToken();
    assertEquals("aıb", st.sval);
}

Example 48 with StreamTokenizer

use of java.io.StreamTokenizer in project siena by mandubian.

the class FullText method addWords.

/**
     * Add all words in the given text to the hash set.
     *
     * @param setting the fulltext settings
     * @param set the hash set
     * @param reader the reader
     */
protected static void addWords(FullTextSettings setting, HashSet<String> set, Reader reader) {
    StreamTokenizer tokenizer = new StreamTokenizer(reader);
    tokenizer.resetSyntax();
    tokenizer.wordChars(' ' + 1, 255);
    for (char ch : " \t\n\r\f+\"*%&/()=?'!,.;:-_#@|^~`{}[]".toCharArray()) {
        tokenizer.whitespaceChars(ch, ch);
    }
    try {
        while (true) {
            int token = tokenizer.nextToken();
            if (token == StreamTokenizer.TT_EOF) {
                break;
            } else if (token == StreamTokenizer.TT_WORD) {
                String word = tokenizer.sval;
                word = setting.convertWord(word);
                if (word != null) {
                    set.add(word);
                }
            }
        }
    } catch (IOException e) {
        throw DbException.convertIOException(e, "Tokenizer error");
    }
}

Example 49 with StreamTokenizer

use of java.io.StreamTokenizer in project intellij-community by JetBrains.

the class InspectionTestUtil method compareDescriptions.

static boolean compareDescriptions(Element reportedProblem, Element expectedProblem) throws Exception {
    String expectedDescription = expectedProblem.getChildText("description");
    String reportedDescription = reportedProblem.getChildText("description");
    if (expectedDescription.equals(reportedDescription))
        return true;
    StreamTokenizer tokenizer = new StreamTokenizer(new CharArrayReader(expectedDescription.toCharArray()));
    tokenizer.quoteChar('\'');
    int idx = 0;
    while (tokenizer.nextToken() != StreamTokenizer.TT_EOF) {
        String word;
        if (tokenizer.sval != null) {
            word = tokenizer.sval;
        } else if (tokenizer.ttype == StreamTokenizer.TT_NUMBER) {
            word = Double.toString(tokenizer.nval);
        } else {
            continue;
        }
        idx = reportedDescription.indexOf(word, idx);
        if (idx == -1)
            return false;
        idx += word.length();
    }
    return true;
}

Example 50 with StreamTokenizer

use of java.io.StreamTokenizer in project XobotOS by xamarin.

the class TypedProperties method initTokenizer.

/**
     * Instantiates a {@link java.io.StreamTokenizer} and sets its syntax tables
     * appropriately for the {@code TypedProperties} file format.
     *
     * @param r The {@code Reader} that the {@code StreamTokenizer} will read from
     * @return a newly-created and initialized {@code StreamTokenizer}
     */
static StreamTokenizer initTokenizer(Reader r) {
    StreamTokenizer st = new StreamTokenizer(r);
    // Treat everything we don't specify as "ordinary".
    st.resetSyntax();
    /* The only non-quoted-string words we'll be reading are:
         * - property names: [._$a-zA-Z0-9]
         * - type names: [a-zS]
         * - number literals: [-0-9.eExXA-Za-z]  ('x' for 0xNNN hex literals. "NaN", "Infinity")
         * - "true" or "false" (case insensitive): [a-zA-Z]
         */
    st.wordChars('0', '9');
    st.wordChars('A', 'Z');
    st.wordChars('a', 'z');
    st.wordChars('_', '_');
    st.wordChars('$', '$');
    st.wordChars('.', '.');
    st.wordChars('-', '-');
    st.wordChars('+', '+');
    // Single-character tokens
    st.ordinaryChar('=');
    // Other special characters
    st.whitespaceChars(' ', ' ');
    st.whitespaceChars('\t', '\t');
    st.whitespaceChars('\n', '\n');
    st.whitespaceChars('\r', '\r');
    st.quoteChar('"');
    // Java-style comments
    st.slashStarComments(true);
    st.slashSlashComments(true);
    return st;
}