Examples with StreamTokenizer java.io.StreamTokenizer used on opensource projects

Example 16 with StreamTokenizer

use of java.io.StreamTokenizer in project zm-mailbox by Zimbra.

the class ZoneInfo2iCalendar method readExtraData.

// Read the file containing PrimaryZone and ZoneMatchScore lines.  PrimaryZone has one argument, a TZID.
// A primary time zone is listed in web client's TZ selection list.  ZoneMatchScore has two arguments, TZID
// and an integer match score.  Score is used to prioritize time zones with identical GMT offsets and
// DST rules (or lack thereof) when looking up a system time zone that best matches a given time zone.
private static void readExtraData(Reader reader) throws IOException, ParseException {
    char dquote = '"';
    StreamTokenizer tokenizer = new StreamTokenizer(reader);
    tokenizer.resetSyntax();
    tokenizer.wordChars(32, 126);
    tokenizer.whitespaceChars(' ', ' ');
    tokenizer.whitespaceChars('\t', '\t');
    tokenizer.whitespaceChars(0, 20);
    tokenizer.commentChar('#');
    tokenizer.quoteChar(dquote);
    tokenizer.eolIsSignificant(true);
    List<String> tokenList = new ArrayList<String>();
    LineType lineType = LineType.UNKNOWN;
    boolean atLineStart = true;
    int ttype;
    // used for empty line detection
    int prevTtype = StreamTokenizer.TT_EOL;
    while ((ttype = tokenizer.nextToken()) != StreamTokenizer.TT_EOF) {
        int lineNum = tokenizer.lineno();
        if (ttype == StreamTokenizer.TT_WORD || ttype == dquote) {
            String token = tokenizer.sval;
            if (atLineStart) {
                lineType = LineType.lookUp(token);
                if (LineType.UNKNOWN.equals(lineType))
                    throw new ParseException("Invalid line type", lineNum);
            } else {
                tokenList.add(token);
            }
            atLineStart = false;
        } else if (ttype == StreamTokenizer.TT_EOL) {
            if (prevTtype == StreamTokenizer.TT_EOL) {
                prevTtype = ttype;
                continue;
            }
            atLineStart = true;
            switch(lineType) {
                case PRIMARYZONE:
                    if (tokenList.size() < 1)
                        throw new ParseException("Not enough fields in a PrimaryZone line", lineNum);
                    String primaryTZID = tokenList.get(0);
                    sPrimaryTZIDs.add(primaryTZID);
                    break;
                case ZONEMATCHSCORE:
                    if (tokenList.size() < 2)
                        throw new ParseException("Not enough fields in a ZoneMatchScore line", lineNum);
                    String zoneName = tokenList.get(0);
                    String zoneMatchScoreStr = tokenList.get(1);
                    int zoneMatchScore = 0;
                    try {
                        zoneMatchScore = Integer.parseInt(zoneMatchScoreStr);
                    } catch (NumberFormatException e) {
                        throw new ParseException("Zone match score must be an integer: " + zoneMatchScoreStr, lineNum);
                    }
                    sMatchScores.put(zoneName, zoneMatchScore);
                    break;
            }
            if (atLineStart) {
                tokenList.clear();
                lineType = LineType.UNKNOWN;
            }
        } else if (ttype == StreamTokenizer.TT_NUMBER) {
            // shouldn't happen
            throw new ParseException("Invalid parser state: TT_NUMBER found", lineNum);
        }
        prevTtype = ttype;
    }
}

Example 17 with StreamTokenizer

use of java.io.StreamTokenizer in project zm-mailbox by Zimbra.

the class ZoneInfoParser method readTzdata.

public void readTzdata(Reader reader) throws IOException, ParseException {
    char dquote = '"';
    StreamTokenizer tokenizer = new StreamTokenizer(reader);
    tokenizer.resetSyntax();
    tokenizer.wordChars(32, 126);
    tokenizer.whitespaceChars(' ', ' ');
    tokenizer.whitespaceChars('\t', '\t');
    tokenizer.whitespaceChars(0, 20);
    tokenizer.commentChar('#');
    tokenizer.quoteChar(dquote);
    tokenizer.eolIsSignificant(true);
    List<String> tokenList = new ArrayList<String>();
    LineType lineType = LineType.UNKNOWN;
    boolean atLineStart = true;
    int ttype;
    // used for empty line detection
    int prevTtype = StreamTokenizer.TT_EOL;
    while ((ttype = tokenizer.nextToken()) != StreamTokenizer.TT_EOF) {
        int lineNum = tokenizer.lineno();
        try {
            if (ttype == StreamTokenizer.TT_WORD || ttype == dquote) {
                String token = tokenizer.sval;
                if (atLineStart) {
                    lineType = LineType.lookUp(token);
                    if (LineType.UNKNOWN.equals(lineType))
                        throw new ParseException("Invalid line type", lineNum);
                } else {
                    tokenList.add(token);
                }
                atLineStart = false;
            } else if (ttype == StreamTokenizer.TT_EOL) {
                if (prevTtype == StreamTokenizer.TT_EOL) {
                    prevTtype = ttype;
                    continue;
                }
                atLineStart = true;
                switch(lineType) {
                    case RULE:
                        RuleLine rl = new RuleLine(tokenList);
                        String rname = rl.getName();
                        Rule rule = mRules.get(rname);
                        if (rule == null) {
                            rule = new Rule(rname);
                            mRules.put(rname, rule);
                        }
                        rule.addRuleLine(rl);
                        break;
                    case ZONE:
                        ZoneLine zl = new ZoneLine(tokenList);
                        String zname = zl.getName();
                        Zone zone = mZones.get(zname);
                        if (zone == null) {
                            zone = new Zone(zname);
                            mZones.put(zname, zone);
                        }
                        zone.addZoneLine(zl);
                        // If Zone line had UNTIL, next line is a continuation of the same Zone.
                        if (zl.hasUntil()) {
                            atLineStart = false;
                            tokenList.clear();
                            tokenList.add(zname);
                        }
                        break;
                    case LINK:
                        if (tokenList.size() < 2)
                            throw new ParseException("Not enough fields in a Link line", lineNum);
                        String real = tokenList.get(0);
                        String alias = tokenList.get(1);
                        if (!alias.equals(real))
                            mLinks.put(alias, real);
                        break;
                    case LEAP:
                        Leap leap = new Leap(tokenList);
                        mLeaps.add(leap);
                        break;
                }
                if (atLineStart) {
                    tokenList.clear();
                    lineType = LineType.UNKNOWN;
                }
            } else if (ttype == StreamTokenizer.TT_NUMBER) {
                // shouldn't happen
                throw new ParseException("Invalid parser state: TT_NUMBER found", lineNum);
            }
            prevTtype = ttype;
        } catch (TZDataParseException e) {
            ParseException pe = new ParseException("Parse error: " + e.getMessage(), lineNum);
            pe.initCause(e);
            throw pe;
        }
    }
}

Example 18 with StreamTokenizer

use of java.io.StreamTokenizer in project android_frameworks_base by DirtyUnicorns.

the class TypedProperties method initTokenizer.

/**
     * Instantiates a {@link java.io.StreamTokenizer} and sets its syntax tables
     * appropriately for the {@code TypedProperties} file format.
     *
     * @param r The {@code Reader} that the {@code StreamTokenizer} will read from
     * @return a newly-created and initialized {@code StreamTokenizer}
     */
static StreamTokenizer initTokenizer(Reader r) {
    StreamTokenizer st = new StreamTokenizer(r);
    // Treat everything we don't specify as "ordinary".
    st.resetSyntax();
    /* The only non-quoted-string words we'll be reading are:
         * - property names: [._$a-zA-Z0-9]
         * - type names: [a-zS]
         * - number literals: [-0-9.eExXA-Za-z]  ('x' for 0xNNN hex literals. "NaN", "Infinity")
         * - "true" or "false" (case insensitive): [a-zA-Z]
         */
    st.wordChars('0', '9');
    st.wordChars('A', 'Z');
    st.wordChars('a', 'z');
    st.wordChars('_', '_');
    st.wordChars('$', '$');
    st.wordChars('.', '.');
    st.wordChars('-', '-');
    st.wordChars('+', '+');
    // Single-character tokens
    st.ordinaryChar('=');
    // Other special characters
    st.whitespaceChars(' ', ' ');
    st.whitespaceChars('\t', '\t');
    st.whitespaceChars('\n', '\n');
    st.whitespaceChars('\r', '\r');
    st.quoteChar('"');
    // Java-style comments
    st.slashStarComments(true);
    st.slashSlashComments(true);
    return st;
}

Example 19 with StreamTokenizer

use of java.io.StreamTokenizer in project jdk8u_jdk by JetBrains.

the class ReadAhead method main.

public static void main(String[] args) throws Exception {
    /* InputStream case */
    test(new StreamTokenizerMaker() {

        public StreamTokenizer create(String input, int limit) {
            return new StreamTokenizer(new LimitedInputStream(input, limit));
        }
    });
    /* Reader case */
    test(new StreamTokenizerMaker() {

        public StreamTokenizer create(String input, int limit) {
            return new StreamTokenizer(new LimitedReader(input, limit));
        }
    });
}

Example 20 with StreamTokenizer

use of java.io.StreamTokenizer in project aima-java by aimacode.

the class Matrix method read.

/**
	 * Read a matrix from a stream. The format is the same the print method, so
	 * printed matrices can be read back in (provided they were printed using US
	 * Locale). Elements are separated by whitespace, all the elements for each
	 * row appear on a single line, the last row is followed by a blank line.
	 * 
	 * @param input
	 *            the input stream.
	 */
@SuppressWarnings("unchecked")
public static Matrix read(BufferedReader input) throws java.io.IOException {
    StreamTokenizer tokenizer = new StreamTokenizer(input);
    // Although StreamTokenizer will parse numbers, it doesn't recognize
    // scientific notation (E or D); however, Double.valueOf does.
    // The strategy here is to disable StreamTokenizer's number parsing.
    // We'll only get whitespace delimited words, EOL's and EOF's.
    // These words should all be numbers, for Double.valueOf to parse.
    tokenizer.resetSyntax();
    tokenizer.wordChars(0, 255);
    tokenizer.whitespaceChars(0, ' ');
    tokenizer.eolIsSignificant(true);
    java.util.Vector v = new java.util.Vector();
    // Ignore initial empty lines
    while (tokenizer.nextToken() == StreamTokenizer.TT_EOL) ;
    if (tokenizer.ttype == StreamTokenizer.TT_EOF)
        throw new java.io.IOException("Unexpected EOF on matrix read.");
    do {
        // Read & store 1st
        v.addElement(Double.valueOf(tokenizer.sval));
    // row.
    } while (tokenizer.nextToken() == StreamTokenizer.TT_WORD);
    // Now we've got the number of columns!
    int n = v.size();
    double[] row = new double[n];
    for (int j = 0; j < n; j++) // extract the elements of the 1st row.
    row[j] = ((Double) v.elementAt(j)).doubleValue();
    v.removeAllElements();
    // Start storing rows instead of columns.
    v.addElement(row);
    while (tokenizer.nextToken() == StreamTokenizer.TT_WORD) {
        // While non-empty lines
        v.addElement(row = new double[n]);
        int j = 0;
        do {
            if (j >= n)
                throw new java.io.IOException("Row " + v.size() + " is too long.");
            row[j++] = Double.valueOf(tokenizer.sval).doubleValue();
        } while (tokenizer.nextToken() == StreamTokenizer.TT_WORD);
        if (j < n)
            throw new java.io.IOException("Row " + v.size() + " is too short.");
    }
    // Now we've got the number of rows.
    int m = v.size();
    double[][] A = new double[m][];
    // copy the rows out of the vector
    v.copyInto(A);
    return new Matrix(A);
}