Examples with StreamTokenizer java.io.StreamTokenizer used on opensource projects

Example 31 with StreamTokenizer

use of java.io.StreamTokenizer in project sonar-web by SonarSource.

the class DoctypeTokenizer method parseToken.

private static void parseToken(DirectiveNode node) {
    String code = node.getCode();
    StreamTokenizer tokenizer = new StreamTokenizer(new StringReader(code));
    tokenizer.quoteChar('"');
    try {
        while (tokenizer.nextToken() != StreamTokenizer.TT_EOF) {
            if (tokenizer.sval != null) {
                if (node.getNodeName() == null) {
                    node.setNodeName(tokenizer.sval);
                } else {
                    node.getAttributes().add(new Attribute(tokenizer.sval));
                }
            }
        }
    } catch (IOException e) {
    // ignore
    }
}

Example 32 with StreamTokenizer

use of java.io.StreamTokenizer in project jop by jop-devel.

the class Jopa method pass2.

/**
*	second pass.
*	generate code and write rom.mif and ram.mif.
*/
public void pass2() {
    StreamTokenizer in = getSt();
    int pc = 0;
    int ji_cnt = 0;
    try {
        FileWriter rom = new FileWriter(dstDir + "rom.mif");
        FileWriter jtbl = new FileWriter(dstDir + "jtbl.vhd");
        BufferedReader inraw = new BufferedReader(new FileReader(srcDir + fname));
        String line;
        //
        //	print rom.mif head
        //
        line = "--\n";
        line += "--\trom.mif\n";
        line += "--\n";
        line += "depth = " + ROM_LEN + ";\n";
        line += "width = " + DATABITS + ";\n";
        line += "\n";
        line += "content\n";
        line += "\n";
        line += "begin\n";
        line += "\n";
        line += "\t[0..1ff] : 080;	-- nop TODO: new instruction\n\n";
        rom.write(line);
        //
        //	print jtbl.vhd head
        //
        line = "--\n";
        line += "--\tjtbl.vhd\n";
        line += "--\n";
        line += "--\tjump table for java bc to jvm address\n";
        line += "--\n";
        line += "--\t\tDONT edit this file!\n";
        line += "--\t\tgenerated by Jopa.java\n";
        line += "--\n";
        line += "\n";
        line += "library ieee;\n";
        line += "use ieee.std_logic_1164.all;\n";
        line += "use ieee.std_logic_arith.all;\n";
        line += "use ieee.std_logic_unsigned.all;\n";
        line += "\n";
        line += "entity jtbl is\n";
        line += "port (\n";
        line += "\tbcode\t: in std_logic_vector(7 downto 0);\n";
        line += "\tint_pend\t: in  std_logic;\n";
        line += "\texc_pend\t: in  std_logic;\n";
        line += "\tq\t\t: out std_logic_vector(" + (ADDRBITS - 1) + " downto 0)\n";
        line += ");\n";
        line += "end jtbl;\n";
        line += "\n";
        line += "--\n";
        line += "--\tunregistered rdbcode\n";
        line += "--\tunregistered dout\n";
        line += "--\n";
        line += "architecture rtl of jtbl is\n";
        line += "\n";
        line += "\tsignal\taddr\t: std_logic_vector(" + (ADDRBITS - 1) + " downto 0);\n";
        line += "\n";
        line += "begin\n";
        line += "\n";
        line += "process(bcode) begin\n";
        line += "\n";
        line += "\tcase bcode is\n";
        line += "\n";
        jtbl.write(line);
        int noim_address = 0;
        int int_address = 0;
        int exc_address = 0;
        while (in.nextToken() != StreamTokenizer.TT_EOF) {
            in.pushBack();
            Line l = getLine(in);
            if (l.jinstr != -1) {
                ++ji_cnt;
                if (JopInstr.name(l.jinstr).equals("sys_int")) {
                    int_address = pc;
                } else if (JopInstr.name(l.jinstr).equals("sys_exc")) {
                    exc_address = pc;
                } else if (JopInstr.name(l.jinstr).equals("sys_noim")) {
                    noim_address = pc;
                } else {
                    jtbl.write("\t\twhen \"" + bin(l.jinstr, 8) + "\" => addr <= \"" + bin(pc, ADDRBITS) + "\";" + "\t--\t" + hex(pc, 4) + "\t" + JopInstr.name(l.jinstr) + "\n");
                }
            }
            line = "\t";
            if (l.instr == null) {
                line += "            ";
            } else {
                //
                //	do the assembling
                //
                int opcode = l.instr.opcode;
                if (l.instr.opdSize != 0) {
                    int opVal = 0;
                    if (l.symVal != null) {
                        Integer i = symMap.get(l.symVal);
                        if (i == null) {
                            error(in, "Symbol " + l.symVal + " not defined");
                        } else {
                            opVal = i.intValue();
                        }
                    } else {
                        opVal = l.intVal;
                    }
                    if (l.instr.name.equals("ldi")) {
                        Integer i = new Integer(opVal);
                        Integer addr;
                        if (constMap.containsKey(i)) {
                            addr = constMap.get(i);
                        } else {
                            addr = new Integer(constMap.size());
                            constMap.put(i, addr);
                            constList.add(i);
                        }
                        opVal = addr.intValue();
                    }
                    int mask = (1 << l.instr.opdSize) - 1;
                    // for branches and jumps opVal points to the target address
                    if (l.instr.jType == JmpType.JMP || l.instr.jType == JmpType.BR) {
                        // relative address
                        opVal = opVal - pc - 1;
                        // check maximum relative offset
                        if (opVal > (mask >> 1) || opVal < (-((mask >> 1) + 1))) {
                            error(in, "jmp/br address too far: " + opVal);
                        }
                        opVal &= mask;
                    }
                    // general check
                    if (opVal > mask || opVal < 0) {
                        error(in, "operand wrong: " + opVal);
                    }
                    // use operand
                    opcode |= opVal & mask;
                }
                if (l.nxt)
                    opcode |= 0x2 << Instruction.INSTLEN;
                if (l.opd)
                    opcode |= 0x1 << Instruction.INSTLEN;
                romData[romLen] = opcode;
                ++romLen;
                line += hex(pc, 4) + " : " + hex(opcode, 3) + ";\t";
                ++pc;
            }
            line += "\t--\t" + inraw.readLine() + "\n";
            rom.write(line);
        // System.out.print(line);
        }
        rom.write("\nend;\n");
        rom.close();
        line = "\n";
        line += "\t\twhen others => addr <= \"" + bin(noim_address, ADDRBITS) + "\";\t\t--\t" + hex(noim_address, 4) + "\tsys_noim\n";
        line += "\tend case;\n";
        line += "end process;\n";
        line += "\n";
        line += "process(int_pend, exc_pend, addr) begin\n";
        line += "\n";
        line += "\tq <= addr;\n";
        line += "\tif exc_pend='1' then\n";
        line += "\t\tq <= \"" + bin(exc_address, ADDRBITS) + "\";\t\t--\t" + hex(exc_address, 4) + "\tsys_exc\n";
        line += "\telsif int_pend='1' then\n";
        line += "\t\tq <= \"" + bin(int_address, ADDRBITS) + "\";\t\t--\t" + hex(int_address, 4) + "\tsys_int\n";
        line += "\tend if;\n";
        line += "end process;\n";
        line += "\n";
        line += "end rtl;\n";
        jtbl.write(line);
        jtbl.close();
        //
        //	print ROM as generic VHDL file
        //
        FileWriter romvhd = new FileWriter(dstDir + "rom.vhd");
        line = "--\n";
        line += "--\trom.vhd\n";
        line += "--\n";
        line += "--\tgeneric VHDL version of ROM\n";
        line += "--\n";
        line += "--\t\tDONT edit this file!\n";
        line += "--\t\tgenerated by Jopa.java\n";
        line += "--\n";
        line += "\n";
        line += "library ieee;\n";
        line += "use ieee.std_logic_1164.all;\n";
        line += "use ieee.std_logic_arith.all;\n";
        line += "use ieee.std_logic_unsigned.all;\n";
        line += "\n";
        line += "entity rom is\n";
        line += "generic (width : integer; addr_width : integer);\t-- for compatibility\n";
        line += "port (\n";
        line += "\tclk\t\t\t: in std_logic;\n";
        line += "\taddress\t\t: in std_logic_vector(" + (ADDRBITS - 1) + " downto 0);\n";
        line += "\tq\t\t\t: out std_logic_vector(" + (DATABITS - 1) + " downto 0)\n";
        line += ");\n";
        line += "end rom;\n";
        line += "\n";
        line += "architecture rtl of rom is\n";
        line += "\n";
        line += "\tsignal areg\t\t: std_logic_vector(" + (ADDRBITS - 1) + " downto 0);\n";
        line += "\tsignal data\t\t: std_logic_vector(" + (DATABITS - 1) + " downto 0);\n";
        line += "\n";
        line += "begin\n";
        line += "\n";
        line += "process(clk) begin\n";
        line += "\n";
        //			line += "\tif falling_edge(clk) then\n";
        //			line += "\t\tareg <= address;\n";
        //			line += "\tend if;\n";
        line += "\tif rising_edge(clk) then\n";
        //			line += "\t\tq <= data;\n";
        line += "\t\tareg <= address;\n";
        line += "\tend if;\n";
        line += "\n";
        line += "end process;\n";
        line += "\n";
        line += "\tq <= data;\n";
        line += "\n";
        line += "process(areg) begin\n";
        line += "\n";
        line += "\tcase areg is\n";
        line += "\n";
        romvhd.write(line);
        for (int i = 0; i < romLen; ++i) {
            romvhd.write("\t\twhen \"" + bin(i, ADDRBITS) + "\" => data <= \"" + bin(romData[i], DATABITS) + "\";");
            romvhd.write("\t-- " + "TODO: comment" + "\n");
        }
        line = "\n";
        line += "\t\twhen others => data <= \"" + bin(0, DATABITS) + "\";\n";
        line += "\tend case;\n";
        line += "end process;\n";
        line += "\n";
        line += "end rtl;\n";
        romvhd.write(line);
        romvhd.close();
        PrintStream rom_mem = new PrintStream(new FileOutputStream(dstDir + "mem_rom.dat"));
        for (int i = 0; i < ROM_LEN; ++i) {
            rom_mem.println(romData[i] + " ");
        }
        rom_mem.close();
        //
        //	Print symbol table as ram.mif and data for the simulation.
        //
        FileWriter ram = new FileWriter(dstDir + "ram.mif");
        for (int i = 0; i < RAM_LEN; ++i) {
            ramData[i] = 0x12345678;
        }
        line = "--\n";
        line += "--\tram.mif\n";
        line += "--\n";
        line += "depth = " + RAM_LEN + ";\n";
        line += "width = 32;\n";
        line += "\n";
        line += "content\n";
        line += "\n";
        line += "begin\n";
        //			line += "\t[0..ff] : 00000000;\n";
        line += "\t[0..ff] : 12345678;\n";
        line += "\n";
        ram.write(line);
        line = "--\n";
        line += "-- " + memcnt + " vars\n";
        line += "--\n\n";
        ram.write(line);
        //
        for (int i = 0; i < varList.size(); ++i) {
            String s = varList.get(i);
            ramData[i] = 0;
            line = "\t";
            line += hex(i, 4) + " : ";
            line += hex(0, 8) + ";\t--\t";
            line += s + "\n";
            ram.write(line);
        }
        line = "--\n";
        line += "-- " + constMap.size() + " consts\n";
        line += "--\n\n";
        ram.write(line);
        if (constMap.size() > VER_ADDR - CONST_ADDR) {
            System.out.println("error: too many constants");
            System.exit(-1);
        }
        //
        for (int i = 0; i < constList.size(); ++i) {
            Integer val = constList.get(i);
            ramData[CONST_ADDR + i] = val.intValue();
            line = "\t";
            line += hex(CONST_ADDR + i, 4) + " : ";
            line += hex(val.intValue(), 8) + ";\t--\t";
            line += val + "\n";
            ram.write(line);
        }
        // check if version is set
        Integer ver = symMap.get("version");
        if (ver == null) {
            error(in, "version not set, setting to -1");
        } else {
            version = ver.intValue();
        }
        ramData[VER_ADDR] = version;
        ramData[VER_ADDR + 1] = 0;
        ram.write("\n\n--\tVersion now in the constant area\n");
        line = "\t";
        line += hex(VER_ADDR, 4) + " : ";
        line += hex(version, 8) + ";\t--\t";
        line += version + "\n";
        ram.write(line);
        line = "\t";
        line = "\t" + hex(VER_ADDR + 1, 4) + " : ";
        line += hex(0, 8) + ";\t--\tfor future use - FPGA type?\n";
        ram.write(line);
        ram.write("\nend;\n");
        ram.close();
        PrintStream ram_mem = new PrintStream(new FileOutputStream(dstDir + "mem_ram.dat"));
        for (int i = 0; i < RAM_LEN; ++i) {
            ram_mem.println(ramData[i] + " ");
        }
        ram_mem.close();
        System.out.println(ji_cnt + " Instructions implemented");
    } catch (IOException e) {
        System.out.println(e.getMessage());
        System.exit(-1);
    }
}

Example 33 with StreamTokenizer

use of java.io.StreamTokenizer in project jop by jop-devel.

the class JavaDown method download.

/**
     * Downloads a JOPized file to JOP
     * 
     * @param file
     * @param serialPort
     * @param usb
     * @param monitor
     * @throws FileNotFoundException
     * @throws IOException
     */
private void download(IPath file, SerialPort serialPort, boolean usb, IProgressMonitor monitor) throws FileNotFoundException, IOException {
    OutputStream serialOut = serialPort.getOutputStream();
    InputStream serialIn = serialPort.getInputStream();
    FileReader fileIn = new FileReader(file.toFile());
    StreamTokenizer tok = new StreamTokenizer(fileIn);
    tok.slashSlashComments(true);
    tok.whitespaceChars(',', ',');
    // Replies to read back
    int numReplies = 0;
    // Total number of words to process
    int nwords = 0;
    // FIXME debug
    serialOut = new FileOutputStream(file.addFileExtension(".down").toFile());
    serialIn = new InputStream() {

        @Override
        public int read() throws IOException {
            return 0;
        }
    };
    for (int numProcessedTokens = 0; tok.nextToken() != StreamTokenizer.TT_EOF; ++numProcessedTokens) {
        // Get the 32 bit word to be sent
        int word = (int) tok.nval;
        // Java code length at index 1 position in .jop
        if (numProcessedTokens == 1) {
            /*sysoutStream.println(word + " words of Java bytecode ("
                        + (word / 256) + " KB)");
                */
            nwords = word;
            monitor.beginTask(IJOPLaunchConfigurationConstants.DOWNLOAD_TASK, nwords);
        }
        for (int i = 0; i < 4; i++) {
            byte b = (byte) (word >> ((3 - i) * 8));
            serialOut.write(b);
            ++numReplies;
            if (!usb) {
                // TODO merge the two branches?
                if (numProcessedTokens == i) {
                    // TODO check reply
                    serialIn.read();
                    --numReplies;
                } else if (serialIn.available() != 0) {
                    serialIn.read();
                    --numReplies;
                }
            }
        }
        // So much work
        monitor.worked(1);
    }
    // Finalize
    while (numReplies > 0) {
        serialIn.read();
        --numReplies;
    }
}

Example 34 with StreamTokenizer

use of java.io.StreamTokenizer in project HanLP by hankcs.

the class Matrix method read.

/**
     * Read a matrix from a stream.  The format is the same the print method,
     * so printed matrices can be read back in (provided they were printed using
     * US Locale).  Elements are separated by
     * whitespace, all the elements for each row appear on a single line,
     * the last row is followed by a blank line.
     *
     * @param input the input stream.
     */
public static Matrix read(BufferedReader input) throws java.io.IOException {
    StreamTokenizer tokenizer = new StreamTokenizer(input);
    // Although StreamTokenizer will parse numbers, it doesn't recognize
    // scientific notation (E or D); however, Double.valueOf does.
    // The strategy here is to disable StreamTokenizer's number parsing.
    // We'll only get whitespace delimited words, EOL's and EOF's.
    // These words should all be numbers, for Double.valueOf to parse.
    tokenizer.resetSyntax();
    tokenizer.wordChars(0, 255);
    tokenizer.whitespaceChars(0, ' ');
    tokenizer.eolIsSignificant(true);
    java.util.Vector<Double> vD = new java.util.Vector<Double>();
    // Ignore initial empty lines
    while (tokenizer.nextToken() == StreamTokenizer.TT_EOL) ;
    if (tokenizer.ttype == StreamTokenizer.TT_EOF)
        throw new java.io.IOException("Unexpected EOF on matrix read.");
    do {
        // Read & store 1st row.
        vD.addElement(Double.valueOf(tokenizer.sval));
    } while (tokenizer.nextToken() == StreamTokenizer.TT_WORD);
    // Now we've got the number of columns!
    int n = vD.size();
    double[] row = new double[n];
    for (// extract the elements of the 1st row.
    int j = 0; // extract the elements of the 1st row.
    j < n; // extract the elements of the 1st row.
    j++) row[j] = vD.elementAt(j).doubleValue();
    java.util.Vector<double[]> v = new java.util.Vector<double[]>();
    // Start storing rows instead of columns.
    v.addElement(row);
    while (tokenizer.nextToken() == StreamTokenizer.TT_WORD) {
        // While non-empty lines
        v.addElement(row = new double[n]);
        int j = 0;
        do {
            if (j >= n)
                throw new java.io.IOException("Row " + v.size() + " is too long.");
            row[j++] = Double.valueOf(tokenizer.sval).doubleValue();
        } while (tokenizer.nextToken() == StreamTokenizer.TT_WORD);
        if (j < n)
            throw new java.io.IOException("Row " + v.size() + " is too short.");
    }
    // Now we've got the number of rows.
    int m = v.size();
    double[][] A = new double[m][];
    // copy the rows out of the vector
    v.copyInto(A);
    return new Matrix(A);
}

Example 35 with StreamTokenizer

use of java.io.StreamTokenizer in project j2objc by google.

the class OldStreamTokenizerTest method test_basicStringTokenizerMethods.

public void test_basicStringTokenizerMethods() throws IOException {
    String str = "Testing 12345 \n alpha \r\n omega";
    String strb = "-3.8 'BLIND mice' \r sEe /* how */ they run";
    StringReader aa = new StringReader(str);
    StringReader ba = new StringReader(strb);
    StreamTokenizer a = new StreamTokenizer(aa);
    StreamTokenizer b = new StreamTokenizer(ba);
    Assert.assertTrue(a.lineno() == 1);
    Assert.assertTrue(a.nextToken() == StreamTokenizer.TT_WORD);
    Assert.assertTrue(a.toString().equals("Token[Testing], line 1"));
    Assert.assertTrue(a.nextToken() == StreamTokenizer.TT_NUMBER);
    Assert.assertTrue(a.toString().equals("Token[n=12345.0], line 1"));
    Assert.assertTrue(a.nextToken() == StreamTokenizer.TT_WORD);
    Assert.assertTrue(a.toString().equals("Token[alpha], line 2"));
    Assert.assertTrue(a.nextToken() == StreamTokenizer.TT_WORD);
    Assert.assertTrue(a.toString().equals("Token[omega], line 3"));
    Assert.assertTrue(a.nextToken() == StreamTokenizer.TT_EOF);
    Assert.assertTrue(a.toString().equals("Token[EOF], line 3"));
    b.commentChar('u');
    b.eolIsSignificant(true);
    b.lowerCaseMode(true);
    b.ordinaryChar('y');
    b.slashStarComments(true);
    Assert.assertTrue(b.nextToken() == StreamTokenizer.TT_NUMBER);
    Assert.assertTrue(b.nval == -3.8);
    Assert.assertTrue(b.toString().equals("Token[n=-3.8], line 1"));
    // '
    Assert.assertTrue(b.nextToken() == 39);
    Assert.assertTrue(b.toString().equals("Token[BLIND mice], line 1"));
    // \n
    Assert.assertTrue(b.nextToken() == 10);
    Assert.assertTrue(b.toString().equals("Token[EOL], line 2"));
    Assert.assertTrue(b.nextToken() == StreamTokenizer.TT_WORD);
    Assert.assertTrue(b.toString().equals("Token[see], line 2"));
    Assert.assertTrue(b.nextToken() == StreamTokenizer.TT_WORD);
    Assert.assertTrue(b.toString().equals("Token[the], line 2"));
    // y
    Assert.assertTrue(b.nextToken() == 121);
    Assert.assertTrue(b.toString().equals("Token['y'], line 2"));
    Assert.assertTrue(b.nextToken() == StreamTokenizer.TT_WORD);
    Assert.assertTrue(b.toString().equals("Token[r], line 2"));
    Assert.assertTrue(b.nextToken() == StreamTokenizer.TT_EOF);
    Assert.assertTrue(b.toString().equals("Token[EOF], line 2"));
}