Examples with Token priv.bajdcc.util.lexer.token.Token used on opensource projects

Example 11 with Token

use of priv.bajdcc.util.lexer.token.Token in project jMiniLang by bajdcc.

the class TestGrammar2 method main.

public static void main(String[] args) {
    // System.out.println("Z -> `a`<,> | B | [`a` `b` Z B]");
    try {
        // Scanner scanner = new Scanner(System.in);
        Grammar grammar = new Grammar("!3 - (28 / (!4 * 7)) * (2 + 4) + 5");
        grammar.addTerminal("i", TokenType.INTEGER, null);
        grammar.addTerminal("NEGATIVE", TokenType.OPERATOR, OperatorType.LOGICAL_NOT);
        grammar.addTerminal("PLUS", TokenType.OPERATOR, OperatorType.PLUS);
        grammar.addTerminal("MINUS", TokenType.OPERATOR, OperatorType.MINUS);
        grammar.addTerminal("TIMES", TokenType.OPERATOR, OperatorType.TIMES);
        grammar.addTerminal("DIVIDE", TokenType.OPERATOR, OperatorType.DIVIDE);
        grammar.addTerminal("LPA", TokenType.OPERATOR, OperatorType.LPARAN);
        grammar.addTerminal("RPA", TokenType.OPERATOR, OperatorType.RPARAN);
        String[] nons = new String[] { "E", "T", "F", "G" };
        for (String non : nons) {
            grammar.addNonTerminal(non);
        }
        grammar.addPatternHandler("1", new IPatternHandler() {

            @Override
            public Object handle(List<Token> tokens, List<Object> symbols) {
                return Integer.parseInt(tokens.get(0).object.toString());
            }

            @Override
            public String getPatternName() {
                return "操作数转换";
            }
        });
        grammar.addPatternHandler("010", new IPatternHandler() {

            @Override
            public Object handle(List<Token> tokens, List<Object> symbols) {
                int lop = (int) symbols.get(0);
                int rop = (int) symbols.get(1);
                Token op = tokens.get(0);
                if (op.kToken == TokenType.OPERATOR) {
                    OperatorType kop = (OperatorType) op.object;
                    switch(kop) {
                        case PLUS:
                            return lop + rop;
                        case MINUS:
                            return lop - rop;
                        case TIMES:
                            return lop * rop;
                        case DIVIDE:
                            if (rop == 0) {
                                return lop;
                            } else {
                                return lop / rop;
                            }
                        default:
                            return 0;
                    }
                } else {
                    return 0;
                }
            }

            @Override
            public String getPatternName() {
                return "二元运算";
            }
        });
        grammar.addPatternHandler("101", new IPatternHandler() {

            @Override
            public Object handle(List<Token> tokens, List<Object> symbols) {
                Token ltok = tokens.get(0);
                Token rtok = tokens.get(1);
                Object exp = symbols.get(0);
                if (ltok.object == OperatorType.LPARAN && rtok.object == OperatorType.RPARAN) {
                    // 判断括号
                    return exp;
                }
                return null;
            }

            @Override
            public String getPatternName() {
                return "括号运算";
            }
        });
        grammar.addPatternHandler("10", new IPatternHandler() {

            @Override
            public Object handle(List<Token> tokens, List<Object> symbols) {
                Token unary = tokens.get(0);
                int op = (int) symbols.get(0);
                if (unary.object == OperatorType.LOGICAL_NOT) {
                    // 判断取反
                    return -op;
                }
                return null;
            }

            @Override
            public String getPatternName() {
                return "一元运算";
            }
        });
        grammar.infer("E -> E @PLUS T | E @MINUS T | T");
        grammar.infer("T -> T @TIMES F | T @DIVIDE F | F");
        grammar.infer("F -> @NEGATIVE G | G");
        grammar.infer("G -> @LPA E @RPA | @i");
        grammar.initialize("E");
        System.out.println(grammar.getPrecedenceString());
        System.out.println(grammar.toString());
        grammar.run();
        System.out.println(grammar.getTokenString());
    // scanner.close();
    } catch (RegexException e) {
        System.err.println(e.getPosition() + "," + e.getMessage());
        e.printStackTrace();
    } catch (SyntaxException e) {
        System.err.println(e.getPosition() + "," + e.getMessage() + " " + e.getInfo());
        e.printStackTrace();
    } catch (GrammarException e) {
        System.err.println(e.getPosition() + "," + e.getMessage() + " " + e.getInfo());
        e.printStackTrace();
    }
}

Example 12 with Token

use of priv.bajdcc.util.lexer.token.Token in project jMiniLang by bajdcc.

the class PrecedenceTable method run.

/**
 * 进行分析
 *
 * @return 计算后的值
 * @throws GrammarException 语法错误
 */
public Object run() throws GrammarException {
    /* 指令堆栈 */
    Stack<PredictionInstruction> spi = new Stack<>();
    /* 数据堆栈 */
    Stack<FixedData> sobj = new Stack<>();
    /* 结束符号进栈 */
    spi.push(new PredictionInstruction(PredictType.EPSILON, -1));
    sobj.push(new FixedData());
    /* 执行步骤顺序 */
    int index = 0;
    /* 输入字符索引 */
    // #为-2
    int input = getTokenId();
    /* 栈顶的终结符索引 */
    int top = -1;
    /* 栈顶的终结符位置 */
    int topIndex = -1;
    while (!(spi.size() == 2 && input == -2)) {
        // 栈层为2且输入为#，退出
        index++;
        println("步骤[" + index + "]");
        println("\t----------------");
        if (input == -1) {
            // 没有找到，非法字符
            err(GrammarError.UNDECLARED);
        }
        if (input == -2 && spi.size() == 1) {
            // 栈为#，输入为#，报错
            err(GrammarError.NULL);
        }
        Token token = iter.ex().token();
        println("\t输入：" + "[" + token + "]");
        if (top != -1 && input != -2 && table[top][input] == PrecedenceType.NULL) {
            err(GrammarError.MISS_PRECEDENCE);
        }
        if (top == -1 || (input != -2 && table[top][input] != PrecedenceType.GT)) {
            /* 栈顶为#，或者top<=input，则直接移进 */
            println("\t移进：[" + token + "]");
            /* 1.指令进栈 */
            spi.push(new PredictionInstruction(PredictType.TERMINAL, input));
            /* 2.数据进栈 */
            sobj.push(new FixedData(token));
            /* 3.保存单词 */
            iter.ex().saveToken();
            /* 4.取下一个单词 */
            iter.scan();
            /* 5.刷新当前输入字符索引 */
            input = getTokenId();
        } else {
            /* 不是移进就是归约 */
            /* 1.从栈顶向下寻找第一个出现LT的终结符 */
            int head;
            int comp_top = top;
            int comp_top_index = topIndex;
            for (head = topIndex - 1; head >= 0; head--) {
                if (spi.get(head).type == PredictType.EPSILON) {
                    // 找到底部#
                    comp_top_index = head + 1;
                    break;
                }
                if (spi.get(head).type == PredictType.TERMINAL) {
                    if (table[spi.get(head).inst][comp_top] == PrecedenceType.LT) {
                        // 找到第一个优先级LT的
                        comp_top_index = head + 1;
                        break;
                    } else if (table[spi.get(head).inst][comp_top] == PrecedenceType.EQ) {
                        // 素短语内部优先级相同
                        comp_top = spi.get(head).inst;
                        comp_top_index = head;
                    }
                }
            }
            // head原来为最左素短语的头，从head+1到栈顶为可归约子串
            int primePhraseCount = spi.size() - comp_top_index;
            /* 2.保存最左素短语 */
            ArrayList<PredictionInstruction> primeInstList = new ArrayList<>();
            ArrayList<FixedData> primeDataList = new ArrayList<>();
            for (int i = 0; i < primePhraseCount; i++) {
                primeInstList.add(0, spi.pop());
                primeDataList.add(0, sobj.pop());
            }
            println("\t----==== 最左素短语模式 ====----");
            String pattern = getPattern(primeInstList);
            println("\t" + pattern + ": " + pattern.replace("0", "[op]").replace("1", "[tok]"));
            println("\t----==== 最左素短语 ====----");
            for (int i = 0; i < primePhraseCount; i++) {
                println("\t" + primeDataList.get(i));
            }
            /* 3.新建指令集和数据集（用于用户级回调） */
            ArrayList<Token> tempTokenList = new ArrayList<>();
            ArrayList<Object> tempObjectList = new ArrayList<>();
            for (int i = 0; i < primePhraseCount; i++) {
                PredictType pt = primeInstList.get(i).type;
                if (pt == PredictType.TERMINAL) {
                    tempTokenList.add(primeDataList.get(i).token);
                } else if (pt == PredictType.NONTERMINAL) {
                    tempObjectList.add(primeDataList.get(i).obj);
                }
            }
            /* 4.寻找定义过的有效的模式，进行归约 */
            IPatternHandler handler = mapPattern.get(pattern);
            if (handler == null) {
                System.err.println("缺少处理模式：" + pattern + ": " + pattern.replace("0", "[op]").replace("1", "[tok]"));
                err(GrammarError.MISS_HANDLER);
            }
            println("\t----==== 处理模式名称 ====----");
            println("\t" + handler.getPatternName());
            /* 5.归约处理 */
            Object result = handler.handle(tempTokenList, tempObjectList);
            println("\t----==== 处理结果 ====----");
            println("\t" + result);
            /* 将结果压栈 */
            /* 6.指令进栈（非终结符进栈） */
            spi.push(new PredictionInstruction(PredictType.NONTERMINAL, -1));
            /* 7.数据进栈（结果进栈） */
            sobj.push(new FixedData(result));
        }
        println("\t----==== 指令堆栈 ====----");
        for (int i = spi.size() - 1; i >= 0; i--) {
            PredictionInstruction pi = spi.get(i);
            switch(pi.type) {
                case NONTERMINAL:
                    println("\t" + i + ": [数据]");
                    break;
                case TERMINAL:
                    println("\t" + i + ": [" + arrTerminals.get(pi.inst).toString() + "]");
                    break;
                case EPSILON:
                    println("\t" + i + ": [" + TokenType.EOF.getName() + "]");
                    break;
                default:
                    break;
            }
        }
        println("\t----==== 数据堆栈 ====----");
        for (int i = sobj.size() - 1; i >= 0; i--) {
            println("\t" + i + ": [" + sobj.get(i) + "]");
        }
        println();
        /* 更新栈顶终结符索引 */
        if (spi.peek().type == PredictType.TERMINAL) {
            top = spi.peek().inst;
            topIndex = spi.size() - 1;
        } else {
            // 若栈顶为非终结符，则第二顶必为终结符
            top = spi.elementAt(spi.size() - 2).inst;
            topIndex = spi.size() - 2;
        }
    }
    println();
    if (sobj.peek().obj == null)
        return sobj.peek().token.object;
    return sobj.peek().obj;
}

Example 13 with Token

use of priv.bajdcc.util.lexer.token.Token in project jMiniLang by bajdcc.

the class TestLexer method main.

public static void main(String[] args) {
    try {
        Scanner scanner = new Scanner(System.in);
        String str = scanner.nextLine();
        scanner.close();
        Lexer lexer = new Lexer(str);
        Token token;
        for (; ; ) {
            token = lexer.scan();
            if (token.kToken == TokenType.EOF) {
                break;
            }
            System.out.println(token.toString());
        }
    } catch (RegexException e) {
        System.err.println(e.getPosition() + "," + e.getMessage());
        e.printStackTrace();
    }
}

Example 14 with Token

use of priv.bajdcc.util.lexer.token.Token in project jMiniLang by bajdcc.

the class Main method main.

public static void main(String[] args) {
    String filename = "E:/vmm.cpp";
    try {
        // 读文件
        BufferedReader br = new BufferedReader(new FileReader(filename));
        String line;
        StringBuilder sb = new StringBuilder();
        while ((line = br.readLine()) != null) {
            sb.append(line).append(System.lineSeparator());
        }
        br.close();
        String context = sb.toString();
        // 词法分析
        Lexer lexer = new Lexer(context);
        lexer.discard(TokenType.COMMENT);
        lexer.discard(TokenType.WHITESPACE);
        lexer.discard(TokenType.ERROR);
        Token token;
        PrintStream ps = new PrintStream(new FileOutputStream("E:\\output.txt"));
        for (; ; ) {
            token = lexer.scan();
            if (token == null) {
                continue;
            }
            if (token.kToken == TokenType.EOF) {
                break;
            }
            ps.println(token.toString());
            ps.flush();
        }
        ps.close();
    } catch (Exception e) {
        System.err.print(e.getMessage());
        e.printStackTrace();
    }
}

Example 15 with Token

use of priv.bajdcc.util.lexer.token.Token in project jMiniLang by bajdcc.

the class TokenAlgorithmCollection method scan.

public Token scan() {
    Token token = new Token();
    token.kToken = TokenType.ERROR;
    if (!iterator.available()) {
        token.kToken = TokenType.EOF;
    } else {
        for (ITokenAlgorithm alg : arrAlgorithms) {
            filterHost.setFilter(alg);
            iterator.translate();
            if (alg.accept(iterator, token))
                return token;
        }
        handler.handleError();
    }
    return token;
}