Examples with StreamTokenizer java.io.StreamTokenizer used on opensource projects

Example 41 with StreamTokenizer

use of java.io.StreamTokenizer in project watson by totemo.

the class CalcCommand method main.

// --------------------------------------------------------------------------
/**
   * Main program for interactive testing.
   */
public static void main(String[] args) throws IOException {
    String commandLine = concat(args);
    StreamTokenizer tokenizer = makeTokenizer(commandLine);
    dump(tokenizer);
    StreamTokenizer tokenizer2 = makeTokenizer(commandLine);
    System.out.printf("%g\n", calculation(tokenizer2));
}

Example 42 with StreamTokenizer

use of java.io.StreamTokenizer in project blueprints by tinkerpop.

the class GMLReader method inputGraph.

/**
     * Load the GML file into the Graph.
     *
     * @param inputGraph       to receive the data
     * @param inputStream      GML file
     * @param defaultEdgeLabel default edge label to be used if not defined in the data
     * @param vertexIdKey      if the id of a vertex is a <data/> property, fetch it from the data property.
     * @param edgeIdKey        if the id of an edge is a <data/> property, fetch it from the data property.
     * @param edgeLabelKey     if the label of an edge is a <data/> property, fetch it from the data property.
     * @throws IOException thrown if the data is not valid
     */
public static void inputGraph(final Graph inputGraph, final InputStream inputStream, final int bufferSize, final String defaultEdgeLabel, final String vertexIdKey, final String edgeIdKey, final String edgeLabelKey) throws IOException {
    final BatchGraph graph = BatchGraph.wrap(inputGraph, bufferSize);
    final Reader r = new BufferedReader(new InputStreamReader(inputStream, Charset.forName("ISO-8859-1")));
    final StreamTokenizer st = new StreamTokenizer(r);
    try {
        st.commentChar(GMLTokens.COMMENT_CHAR);
        st.ordinaryChar('[');
        st.ordinaryChar(']');
        final String stringCharacters = "/\\(){}<>!£$%^&*-+=,.?:;@_`|~";
        for (int i = 0; i < stringCharacters.length(); i++) {
            st.wordChars(stringCharacters.charAt(i), stringCharacters.charAt(i));
        }
        new GMLParser(graph, defaultEdgeLabel, vertexIdKey, edgeIdKey, edgeLabelKey).parse(st);
        graph.commit();
    } catch (IOException e) {
        throw new IOException("GML malformed line number " + st.lineno() + ": ", e);
    } finally {
        r.close();
    }
}

Example 43 with StreamTokenizer

use of java.io.StreamTokenizer in project android_frameworks_base by ParanoidAndroid.

the class TypedProperties method initTokenizer.

/**
     * Instantiates a {@link java.io.StreamTokenizer} and sets its syntax tables
     * appropriately for the {@code TypedProperties} file format.
     *
     * @param r The {@code Reader} that the {@code StreamTokenizer} will read from
     * @return a newly-created and initialized {@code StreamTokenizer}
     */
static StreamTokenizer initTokenizer(Reader r) {
    StreamTokenizer st = new StreamTokenizer(r);
    // Treat everything we don't specify as "ordinary".
    st.resetSyntax();
    /* The only non-quoted-string words we'll be reading are:
         * - property names: [._$a-zA-Z0-9]
         * - type names: [a-zS]
         * - number literals: [-0-9.eExXA-Za-z]  ('x' for 0xNNN hex literals. "NaN", "Infinity")
         * - "true" or "false" (case insensitive): [a-zA-Z]
         */
    st.wordChars('0', '9');
    st.wordChars('A', 'Z');
    st.wordChars('a', 'z');
    st.wordChars('_', '_');
    st.wordChars('$', '$');
    st.wordChars('.', '.');
    st.wordChars('-', '-');
    st.wordChars('+', '+');
    // Single-character tokens
    st.ordinaryChar('=');
    // Other special characters
    st.whitespaceChars(' ', ' ');
    st.whitespaceChars('\t', '\t');
    st.whitespaceChars('\n', '\n');
    st.whitespaceChars('\r', '\r');
    st.quoteChar('"');
    // Java-style comments
    st.slashStarComments(true);
    st.slashSlashComments(true);
    return st;
}

Example 44 with StreamTokenizer

use of java.io.StreamTokenizer in project android_frameworks_base by ParanoidAndroid.

the class TypedProperties method parse.

/**
     * Parses the data in the reader.
     *
     * @param r The {@code Reader} containing input data to parse
     * @param map The {@code Map} to insert parameter values into
     * @throws ParseException if the input data is malformed
     * @throws IOException if there is a problem reading from the {@code Reader}
     */
static void parse(Reader r, Map<String, Object> map) throws ParseException, IOException {
    final StreamTokenizer st = initTokenizer(r);
    /* A property name must be a valid fully-qualified class + package name.
         * We don't support Unicode, though.
         */
    final String identifierPattern = "[a-zA-Z_$][0-9a-zA-Z_$]*";
    final Pattern propertyNamePattern = Pattern.compile("(" + identifierPattern + "\\.)*" + identifierPattern);
    while (true) {
        int token;
        // Read the next token, which is either the type or EOF.
        token = st.nextToken();
        if (token == StreamTokenizer.TT_EOF) {
            break;
        }
        if (token != StreamTokenizer.TT_WORD) {
            throw new ParseException(st, "type name");
        }
        final int type = interpretType(st.sval);
        if (type == TYPE_ERROR) {
            throw new ParseException(st, "valid type name");
        }
        st.sval = null;
        if (type == TYPE_UNSET) {
            // Expect '('.
            token = st.nextToken();
            if (token != '(') {
                throw new ParseException(st, "'('");
            }
        }
        // Read the property name.
        token = st.nextToken();
        if (token != StreamTokenizer.TT_WORD) {
            throw new ParseException(st, "property name");
        }
        final String propertyName = st.sval;
        if (!propertyNamePattern.matcher(propertyName).matches()) {
            throw new ParseException(st, "valid property name");
        }
        st.sval = null;
        if (type == TYPE_UNSET) {
            // Expect ')'.
            token = st.nextToken();
            if (token != ')') {
                throw new ParseException(st, "')'");
            }
            map.remove(propertyName);
        } else {
            // Expect '='.
            token = st.nextToken();
            if (token != '=') {
                throw new ParseException(st, "'='");
            }
            // Read a value of the appropriate type, and insert into the map.
            final Object value = parseValue(st, type);
            final Object oldValue = map.remove(propertyName);
            if (oldValue != null) {
                //       the same property is defined with a different type.
                if (value.getClass() != oldValue.getClass()) {
                    throw new ParseException(st, "(property previously declared as a different type)");
                }
            }
            map.put(propertyName, value);
        }
        // Expect ';'.
        token = st.nextToken();
        if (token != ';') {
            throw new ParseException(st, "';'");
        }
    }
}

Example 45 with StreamTokenizer

use of java.io.StreamTokenizer in project robovm by robovm.

the class OldStreamTokenizerTest method test_basicStringTokenizerMethods.

public void test_basicStringTokenizerMethods() throws IOException {
    String str = "Testing 12345 \n alpha \r\n omega";
    String strb = "-3.8 'BLIND mice' \r sEe /* how */ they run";
    StringReader aa = new StringReader(str);
    StringReader ba = new StringReader(strb);
    StreamTokenizer a = new StreamTokenizer(aa);
    StreamTokenizer b = new StreamTokenizer(ba);
    Assert.assertTrue(a.lineno() == 1);
    Assert.assertTrue(a.nextToken() == StreamTokenizer.TT_WORD);
    Assert.assertTrue(a.toString().equals("Token[Testing], line 1"));
    Assert.assertTrue(a.nextToken() == StreamTokenizer.TT_NUMBER);
    Assert.assertTrue(a.toString().equals("Token[n=12345.0], line 1"));
    Assert.assertTrue(a.nextToken() == StreamTokenizer.TT_WORD);
    Assert.assertTrue(a.toString().equals("Token[alpha], line 2"));
    Assert.assertTrue(a.nextToken() == StreamTokenizer.TT_WORD);
    Assert.assertTrue(a.toString().equals("Token[omega], line 3"));
    Assert.assertTrue(a.nextToken() == StreamTokenizer.TT_EOF);
    Assert.assertTrue(a.toString().equals("Token[EOF], line 3"));
    b.commentChar('u');
    b.eolIsSignificant(true);
    b.lowerCaseMode(true);
    b.ordinaryChar('y');
    b.slashStarComments(true);
    Assert.assertTrue(b.nextToken() == StreamTokenizer.TT_NUMBER);
    Assert.assertTrue(b.nval == -3.8);
    Assert.assertTrue(b.toString().equals("Token[n=-3.8], line 1"));
    // '
    Assert.assertTrue(b.nextToken() == 39);
    Assert.assertTrue(b.toString().equals("Token[BLIND mice], line 1"));
    // \n
    Assert.assertTrue(b.nextToken() == 10);
    Assert.assertTrue(b.toString().equals("Token[EOL], line 2"));
    Assert.assertTrue(b.nextToken() == StreamTokenizer.TT_WORD);
    Assert.assertTrue(b.toString().equals("Token[see], line 2"));
    Assert.assertTrue(b.nextToken() == StreamTokenizer.TT_WORD);
    Assert.assertTrue(b.toString().equals("Token[the], line 2"));
    // y
    Assert.assertTrue(b.nextToken() == 121);
    Assert.assertTrue(b.toString().equals("Token['y'], line 2"));
    Assert.assertTrue(b.nextToken() == StreamTokenizer.TT_WORD);
    Assert.assertTrue(b.toString().equals("Token[r], line 2"));
    Assert.assertTrue(b.nextToken() == StreamTokenizer.TT_EOF);
    Assert.assertTrue(b.toString().equals("Token[EOF], line 2"));
}