Examples with Token priv.bajdcc.LL1.syntax.token.Token used on opensource projects

Example 1 with Token

use of priv.bajdcc.LL1.syntax.token.Token in project jMiniLang by bajdcc.

the class Syntax method getParagraphString.

/**
 * 获得段落式描述
 * @return 段落式描述
 */
public String getParagraphString() {
    StringBuilder sb = new StringBuilder();
    /* 起始符号 */
    sb.append("#### 起始符号 ####");
    sb.append(System.lineSeparator());
    sb.append(beginRuleName);
    sb.append(System.lineSeparator());
    /* 终结符 */
    sb.append("#### 终结符 ####");
    sb.append(System.lineSeparator());
    for (TokenExp exp : arrTerminals) {
        sb.append(exp.toString());
        sb.append(System.lineSeparator());
    }
    /* 非终结符 */
    sb.append("#### 非终结符 ####");
    sb.append(System.lineSeparator());
    for (RuleExp exp : arrNonTerminals) {
        sb.append(exp.toString());
        sb.append(System.lineSeparator());
    }
    /* 推导规则 */
    sb.append("#### 文法产生式 ####");
    sb.append(System.lineSeparator());
    for (RuleExp exp : arrNonTerminals) {
        for (RuleItem item : exp.rule.arrRules) {
            /* 规则正文 */
            sb.append(getSingleString(exp.name, item.expression));
            sb.append(System.lineSeparator());
            /* First集合 */
            sb.append("\t--== First ==--");
            sb.append(System.lineSeparator());
            for (TokenExp token : item.arrFirstSetTokens) {
                sb.append("\t\t").append(token.toString());
                sb.append(System.lineSeparator());
            }
            sb.append("\t--== Follow ==--");
            sb.append(System.lineSeparator());
            for (TokenExp token : item.parent.arrFollows) {
                sb.append("\t\t").append(token.toString());
                sb.append(System.lineSeparator());
            }
        }
    }
    return sb.toString();
}

Example 2 with Token

use of priv.bajdcc.LL1.syntax.token.Token in project jMiniLang by bajdcc.

the class PrecedenceTable method run.

/**
 * 进行分析
 *
 * @return 计算后的值
 * @throws GrammarException 语法错误
 */
public Object run() throws GrammarException {
    /* 指令堆栈 */
    Stack<PredictionInstruction> spi = new Stack<>();
    /* 数据堆栈 */
    Stack<FixedData> sobj = new Stack<>();
    /* 结束符号进栈 */
    spi.push(new PredictionInstruction(PredictType.EPSILON, -1));
    sobj.push(new FixedData());
    /* 执行步骤顺序 */
    int index = 0;
    /* 输入字符索引 */
    // #为-2
    int input = getTokenId();
    /* 栈顶的终结符索引 */
    int top = -1;
    /* 栈顶的终结符位置 */
    int topIndex = -1;
    while (!(spi.size() == 2 && input == -2)) {
        // 栈层为2且输入为#，退出
        index++;
        println("步骤[" + index + "]");
        println("\t----------------");
        if (input == -1) {
            // 没有找到，非法字符
            err(GrammarError.UNDECLARED);
        }
        if (input == -2 && spi.size() == 1) {
            // 栈为#，输入为#，报错
            err(GrammarError.NULL);
        }
        Token token = iter.ex().token();
        println("\t输入：" + "[" + token + "]");
        if (top != -1 && input != -2 && table[top][input] == PrecedenceType.NULL) {
            err(GrammarError.MISS_PRECEDENCE);
        }
        if (top == -1 || (input != -2 && table[top][input] != PrecedenceType.GT)) {
            /* 栈顶为#，或者top<=input，则直接移进 */
            println("\t移进：[" + token + "]");
            /* 1.指令进栈 */
            spi.push(new PredictionInstruction(PredictType.TERMINAL, input));
            /* 2.数据进栈 */
            sobj.push(new FixedData(token));
            /* 3.保存单词 */
            iter.ex().saveToken();
            /* 4.取下一个单词 */
            iter.scan();
            /* 5.刷新当前输入字符索引 */
            input = getTokenId();
        } else {
            /* 不是移进就是归约 */
            /* 1.从栈顶向下寻找第一个出现LT的终结符 */
            int head;
            int comp_top = top;
            int comp_top_index = topIndex;
            for (head = topIndex - 1; head >= 0; head--) {
                if (spi.get(head).type == PredictType.EPSILON) {
                    // 找到底部#
                    comp_top_index = head + 1;
                    break;
                }
                if (spi.get(head).type == PredictType.TERMINAL) {
                    if (table[spi.get(head).inst][comp_top] == PrecedenceType.LT) {
                        // 找到第一个优先级LT的
                        comp_top_index = head + 1;
                        break;
                    } else if (table[spi.get(head).inst][comp_top] == PrecedenceType.EQ) {
                        // 素短语内部优先级相同
                        comp_top = spi.get(head).inst;
                        comp_top_index = head;
                    }
                }
            }
            // head原来为最左素短语的头，从head+1到栈顶为可归约子串
            int primePhraseCount = spi.size() - comp_top_index;
            /* 2.保存最左素短语 */
            ArrayList<PredictionInstruction> primeInstList = new ArrayList<>();
            ArrayList<FixedData> primeDataList = new ArrayList<>();
            for (int i = 0; i < primePhraseCount; i++) {
                primeInstList.add(0, spi.pop());
                primeDataList.add(0, sobj.pop());
            }
            println("\t----==== 最左素短语模式 ====----");
            String pattern = getPattern(primeInstList);
            println("\t" + pattern + ": " + pattern.replace("0", "[op]").replace("1", "[tok]"));
            println("\t----==== 最左素短语 ====----");
            for (int i = 0; i < primePhraseCount; i++) {
                println("\t" + primeDataList.get(i));
            }
            /* 3.新建指令集和数据集（用于用户级回调） */
            ArrayList<Token> tempTokenList = new ArrayList<>();
            ArrayList<Object> tempObjectList = new ArrayList<>();
            for (int i = 0; i < primePhraseCount; i++) {
                PredictType pt = primeInstList.get(i).type;
                if (pt == PredictType.TERMINAL) {
                    tempTokenList.add(primeDataList.get(i).token);
                } else if (pt == PredictType.NONTERMINAL) {
                    tempObjectList.add(primeDataList.get(i).obj);
                }
            }
            /* 4.寻找定义过的有效的模式，进行归约 */
            IPatternHandler handler = mapPattern.get(pattern);
            if (handler == null) {
                System.err.println("缺少处理模式：" + pattern + ": " + pattern.replace("0", "[op]").replace("1", "[tok]"));
                err(GrammarError.MISS_HANDLER);
            }
            println("\t----==== 处理模式名称 ====----");
            println("\t" + handler.getPatternName());
            /* 5.归约处理 */
            Object result = handler.handle(tempTokenList, tempObjectList);
            println("\t----==== 处理结果 ====----");
            println("\t" + result);
            /* 将结果压栈 */
            /* 6.指令进栈（非终结符进栈） */
            spi.push(new PredictionInstruction(PredictType.NONTERMINAL, -1));
            /* 7.数据进栈（结果进栈） */
            sobj.push(new FixedData(result));
        }
        println("\t----==== 指令堆栈 ====----");
        for (int i = spi.size() - 1; i >= 0; i--) {
            PredictionInstruction pi = spi.get(i);
            switch(pi.type) {
                case NONTERMINAL:
                    println("\t" + i + ": [数据]");
                    break;
                case TERMINAL:
                    println("\t" + i + ": [" + arrTerminals.get(pi.inst).toString() + "]");
                    break;
                case EPSILON:
                    println("\t" + i + ": [" + TokenType.EOF.getName() + "]");
                    break;
                default:
                    break;
            }
        }
        println("\t----==== 数据堆栈 ====----");
        for (int i = sobj.size() - 1; i >= 0; i--) {
            println("\t" + i + ": [" + sobj.get(i) + "]");
        }
        println();
        /* 更新栈顶终结符索引 */
        if (spi.peek().type == PredictType.TERMINAL) {
            top = spi.peek().inst;
            topIndex = spi.size() - 1;
        } else {
            // 若栈顶为非终结符，则第二顶必为终结符
            top = spi.elementAt(spi.size() - 2).inst;
            topIndex = spi.size() - 2;
        }
    }
    println();
    if (sobj.peek().obj == null)
        return sobj.peek().token.object;
    return sobj.peek().obj;
}

Example 3 with Token

use of priv.bajdcc.LL1.syntax.token.Token in project jMiniLang by bajdcc.

the class SelectSetSolver method visitBegin.

@Override
public void visitBegin(TokenExp node, VisitBag bag) {
    bag.bVisitChildren = false;
    bag.bVisitEnd = false;
    if (node.kType == TokenType.EOF) {
        // Epsilon
        // 空串，则指令集为空（不压入新状态）
        addRule();
        for (TokenExp token : getFollow()) {
            // 有空串，添加Follow集
            setCellToRuleId(token.id);
        }
        firstSymbol = false;
    } else if (firstSymbol) {
        // 需要添加指令集
        addRule();
        setCellToRuleId(node.id);
        firstSymbol = false;
        insertSymbol = true;
    }
    if (insertSymbol) {
        addInstToRule(PredictType.TERMINAL, node.id);
    }
    if (node.kType != TokenType.EOF) {
        epsilon = false;
    }
}

Example 4 with Token

use of priv.bajdcc.LL1.syntax.token.Token in project jMiniLang by bajdcc.

the class TestSyntaxLexer method main.

public static void main(String[] args) {
    try {
        Scanner scanner = new Scanner(System.in);
        String str = scanner.nextLine();
        SyntaxLexer lexer = new SyntaxLexer();
        lexer.setContext(str);
        Token token;
        for (; ; ) {
            token = lexer.nextToken();
            if (token.kToken == TokenType.EOF) {
                break;
            }
            System.out.println(token.toString());
        }
        scanner.close();
    } catch (RegexException e) {
        System.err.println(e.getPosition() + "," + e.getMessage());
        e.printStackTrace();
    }
}

Example 5 with Token

use of priv.bajdcc.LL1.syntax.token.Token in project jMiniLang by bajdcc.

the class TestSyntaxLexer method main.

public static void main(String[] args) {
    try {
        Scanner scanner = new Scanner(System.in);
        String str = scanner.nextLine();
        SyntaxLexer lexer = new SyntaxLexer();
        lexer.setContext(str);
        Token token;
        for (; ; ) {
            token = lexer.nextToken();
            if (token.kToken == TokenType.EOF) {
                break;
            }
            System.out.println(token.toString());
        }
        scanner.close();
    } catch (RegexException e) {
        System.err.println(e.getPosition() + "," + e.getMessage());
        e.printStackTrace();
    }
}