Examples with THIS org.apache.groovy.parser.antlr4.GroovyLexer.THIS used on opensource projects

Example 1 with THIS

use of org.apache.groovy.parser.antlr4.GroovyLexer.THIS in project groovy by apache.

the class AstBuilder method visitClassDeclaration.

@Override
public ClassNode visitClassDeclaration(final ClassDeclarationContext ctx) {
    String packageName = Optional.ofNullable(moduleNode.getPackageName()).orElse("");
    String className = this.visitIdentifier(ctx.identifier());
    if (VAR_STR.equals(className)) {
        throw createParsingFailedException("var cannot be used for type declarations", ctx.identifier());
    }
    boolean isAnnotation = asBoolean(ctx.AT());
    if (isAnnotation) {
        if (asBoolean(ctx.typeParameters())) {
            throw createParsingFailedException("annotation declaration cannot have type parameters", ctx.typeParameters());
        }
        if (asBoolean(ctx.EXTENDS())) {
            throw createParsingFailedException("No extends clause allowed for annotation declaration", ctx.EXTENDS());
        }
        if (asBoolean(ctx.IMPLEMENTS())) {
            throw createParsingFailedException("No implements clause allowed for annotation declaration", ctx.IMPLEMENTS());
        }
    }
    boolean isEnum = asBoolean(ctx.ENUM());
    if (isEnum) {
        if (asBoolean(ctx.typeParameters())) {
            throw createParsingFailedException("enum declaration cannot have type parameters", ctx.typeParameters());
        }
        if (asBoolean(ctx.EXTENDS())) {
            throw createParsingFailedException("No extends clause allowed for enum declaration", ctx.EXTENDS());
        }
    }
    boolean isInterface = (asBoolean(ctx.INTERFACE()) && !isAnnotation);
    if (isInterface) {
        if (asBoolean(ctx.IMPLEMENTS())) {
            throw createParsingFailedException("No implements clause allowed for interface declaration", ctx.IMPLEMENTS());
        }
    }
    List<ModifierNode> modifierNodeList = ctx.getNodeMetaData(TYPE_DECLARATION_MODIFIERS);
    Objects.requireNonNull(modifierNodeList, "modifierNodeList should not be null");
    ModifierManager modifierManager = new ModifierManager(this, modifierNodeList);
    Optional<ModifierNode> finalModifierNodeOptional = modifierManager.get(FINAL);
    Optional<ModifierNode> sealedModifierNodeOptional = modifierManager.get(SEALED);
    Optional<ModifierNode> nonSealedModifierNodeOptional = modifierManager.get(NON_SEALED);
    boolean isFinal = finalModifierNodeOptional.isPresent();
    boolean isSealed = sealedModifierNodeOptional.isPresent();
    boolean isNonSealed = nonSealedModifierNodeOptional.isPresent();
    boolean isRecord = asBoolean(ctx.RECORD());
    boolean hasRecordHeader = asBoolean(ctx.formalParameters());
    if (isRecord) {
        if (asBoolean(ctx.EXTENDS())) {
            throw createParsingFailedException("No extends clause allowed for record declaration", ctx.EXTENDS());
        }
        if (!hasRecordHeader) {
            throw createParsingFailedException("header declaration of record is expected", ctx.identifier());
        }
        if (isSealed) {
            throw createParsingFailedException("`sealed` is not allowed for record declaration", sealedModifierNodeOptional.get());
        }
        if (isNonSealed) {
            throw createParsingFailedException("`non-sealed` is not allowed for record declaration", nonSealedModifierNodeOptional.get());
        }
    } else {
        if (hasRecordHeader) {
            throw createParsingFailedException("header declaration is only allowed for record declaration", ctx.formalParameters());
        }
    }
    if (isSealed && isNonSealed) {
        throw createParsingFailedException("type cannot be defined with both `sealed` and `non-sealed`", nonSealedModifierNodeOptional.get());
    }
    if (isFinal && (isSealed || isNonSealed)) {
        throw createParsingFailedException("type cannot be defined with both " + (isSealed ? "`sealed`" : "`non-sealed`") + " and `final`", finalModifierNodeOptional.get());
    }
    if ((isAnnotation || isEnum) && (isSealed || isNonSealed)) {
        ModifierNode mn = isSealed ? sealedModifierNodeOptional.get() : nonSealedModifierNodeOptional.get();
        throw createParsingFailedException("modifier `" + mn.getText() + "` is not allowed for " + (isEnum ? "enum" : "annotation definition"), mn);
    }
    boolean hasPermits = asBoolean(ctx.PERMITS());
    if (!isSealed && hasPermits) {
        throw createParsingFailedException("only sealed type declarations should have `permits` clause", ctx);
    }
    int modifiers = modifierManager.getClassModifiersOpValue();
    boolean syntheticPublic = ((modifiers & Opcodes.ACC_SYNTHETIC) != 0);
    modifiers &= ~Opcodes.ACC_SYNTHETIC;
    ClassNode classNode, outerClass = classNodeStack.peek();
    if (isEnum) {
        classNode = EnumHelper.makeEnumNode(asBoolean(outerClass) ? className : packageName + className, modifiers, null, outerClass);
    } else if (asBoolean(outerClass)) {
        if (outerClass.isInterface())
            modifiers |= Opcodes.ACC_STATIC;
        classNode = new InnerClassNode(outerClass, outerClass.getName() + "$" + className, modifiers, ClassHelper.OBJECT_TYPE.getPlainNodeReference());
    } else {
        classNode = new ClassNode(packageName + className, modifiers, ClassHelper.OBJECT_TYPE.getPlainNodeReference());
    }
    configureAST(classNode, ctx);
    classNode.setSyntheticPublic(syntheticPublic);
    classNode.setGenericsTypes(this.visitTypeParameters(ctx.typeParameters()));
    boolean isInterfaceWithDefaultMethods = (isInterface && this.containsDefaultMethods(ctx));
    if (isSealed) {
        AnnotationNode sealedAnnotationNode = new AnnotationNode(ClassHelper.makeCached(Sealed.class));
        if (asBoolean(ctx.ps)) {
            ListExpression permittedSubclassesListExpression = listX(Arrays.stream(this.visitTypeList(ctx.ps)).map(ClassExpression::new).collect(Collectors.toList()));
            sealedAnnotationNode.setMember("permittedSubclasses", permittedSubclassesListExpression);
            configureAST(sealedAnnotationNode, ctx.PERMITS());
            sealedAnnotationNode.setNodeMetaData("permits", true);
        }
        classNode.addAnnotation(sealedAnnotationNode);
    } else if (isNonSealed) {
        classNode.addAnnotation(new AnnotationNode(ClassHelper.makeCached(NonSealed.class)));
    }
    if (isInterfaceWithDefaultMethods || asBoolean(ctx.TRAIT())) {
        classNode.addAnnotation(new AnnotationNode(ClassHelper.makeCached(Trait.class)));
    }
    if (isRecord) {
        classNode.addAnnotation(new AnnotationNode(RECORD_TYPE_CLASS));
    }
    classNode.addAnnotations(modifierManager.getAnnotations());
    if (isInterfaceWithDefaultMethods) {
        classNode.putNodeMetaData(IS_INTERFACE_WITH_DEFAULT_METHODS, Boolean.TRUE);
    }
    classNode.putNodeMetaData(CLASS_NAME, className);
    if (asBoolean(ctx.CLASS()) || asBoolean(ctx.TRAIT())) {
        if (asBoolean(ctx.scs)) {
            ClassNode[] scs = this.visitTypeList(ctx.scs);
            if (scs.length > 1) {
                throw createParsingFailedException("Cannot extend multiple classes", ctx.EXTENDS());
            }
            classNode.setSuperClass(scs[0]);
        }
        classNode.setInterfaces(this.visitTypeList(ctx.is));
        this.initUsingGenerics(classNode);
    } else if (isInterfaceWithDefaultMethods) {
        // GROOVY-9259
        classNode.setInterfaces(this.visitTypeList(ctx.scs));
        this.initUsingGenerics(classNode);
    } else if (isInterface) {
        classNode.setModifiers(classNode.getModifiers() | Opcodes.ACC_INTERFACE | Opcodes.ACC_ABSTRACT);
        classNode.setInterfaces(this.visitTypeList(ctx.scs));
        this.initUsingGenerics(classNode);
        this.hackMixins(classNode);
    } else if (isEnum || isRecord) {
        classNode.setInterfaces(this.visitTypeList(ctx.is));
        this.initUsingGenerics(classNode);
        if (isRecord) {
            transformRecordHeaderToProperties(ctx, classNode);
        }
    } else if (isAnnotation) {
        classNode.setModifiers(classNode.getModifiers() | Opcodes.ACC_INTERFACE | Opcodes.ACC_ABSTRACT | Opcodes.ACC_ANNOTATION);
        classNode.addInterface(ClassHelper.Annotation_TYPE);
        this.hackMixins(classNode);
    } else {
        throw createParsingFailedException("Unsupported class declaration: " + ctx.getText(), ctx);
    }
    // have here to ensure it won't be the inner class
    if (asBoolean(ctx.CLASS()) || asBoolean(ctx.TRAIT())) {
        classNodeList.add(classNode);
    }
    classNodeStack.push(classNode);
    ctx.classBody().putNodeMetaData(CLASS_DECLARATION_CLASS_NODE, classNode);
    this.visitClassBody(ctx.classBody());
    if (isRecord) {
        Optional<FieldNode> fieldNodeOptional = classNode.getFields().stream().filter(f -> !isTrue(f, IS_RECORD_GENERATED) && !f.isStatic()).findFirst();
        if (fieldNodeOptional.isPresent()) {
            createParsingFailedException("Instance field is not allowed in `record`", fieldNodeOptional.get());
        }
    }
    classNodeStack.pop();
    if (!(asBoolean(ctx.CLASS()) || asBoolean(ctx.TRAIT()))) {
        classNodeList.add(classNode);
    }
    groovydocManager.handle(classNode, ctx);
    return classNode;
}

Example 2 with THIS

use of org.apache.groovy.parser.antlr4.GroovyLexer.THIS in project groovy by apache.

the class AstBuilder method visitGstring.

// } literal ---------------------------------------------------------------
// gstring { ---------------------------------------------------------------
@Override
public GStringExpression visitGstring(final GstringContext ctx) {
    final List<ConstantExpression> stringLiteralList = new LinkedList<>();
    final String begin = ctx.GStringBegin().getText();
    final String beginQuotation = beginQuotation(begin);
    stringLiteralList.add(configureAST(new ConstantExpression(parseGStringBegin(ctx, beginQuotation)), ctx.GStringBegin()));
    List<ConstantExpression> partStrings = ctx.GStringPart().stream().map(e -> configureAST(new ConstantExpression(parseGStringPart(e, beginQuotation)), e)).collect(Collectors.toList());
    stringLiteralList.addAll(partStrings);
    stringLiteralList.add(configureAST(new ConstantExpression(parseGStringEnd(ctx, beginQuotation)), ctx.GStringEnd()));
    List<Expression> values = ctx.gstringValue().stream().map(this::visitGstringValue).collect(Collectors.toList());
    StringBuilder verbatimText = new StringBuilder(ctx.getText().length());
    for (int i = 0, n = stringLiteralList.size(), s = values.size(); i < n; i += 1) {
        verbatimText.append(stringLiteralList.get(i).getValue());
        if (i == s) {
            continue;
        }
        Expression value = values.get(i);
        if (!asBoolean(value)) {
            continue;
        }
        boolean isVariableExpression = value instanceof VariableExpression;
        verbatimText.append(DOLLAR_STR);
        if (!isVariableExpression)
            verbatimText.append('{');
        verbatimText.append(value.getText());
        if (!isVariableExpression)
            verbatimText.append('}');
    }
    return configureAST(new GStringExpression(verbatimText.toString(), stringLiteralList, values), ctx);
}

Example 3 with THIS

use of org.apache.groovy.parser.antlr4.GroovyLexer.THIS in project groovy by apache.

the class AstBuilder method visitTryCatchStatement.

@Override
public Statement visitTryCatchStatement(final TryCatchStatementContext ctx) {
    boolean resourcesExists = asBoolean(ctx.resources());
    boolean catchExists = asBoolean(ctx.catchClause());
    boolean finallyExists = asBoolean(ctx.finallyBlock());
    if (!(resourcesExists || catchExists || finallyExists)) {
        throw createParsingFailedException("Either a catch or finally clause or both is required for a try-catch-finally statement", ctx);
    }
    TryCatchStatement tryCatchStatement = new TryCatchStatement((Statement) this.visit(ctx.block()), this.visitFinallyBlock(ctx.finallyBlock()));
    if (resourcesExists) {
        this.visitResources(ctx.resources()).forEach(tryCatchStatement::addResource);
    }
    ctx.catchClause().stream().map(this::visitCatchClause).reduce(new LinkedList<>(), (r, e) -> {
        // merge several LinkedList<CatchStatement> instances into one LinkedList<CatchStatement> instance
        r.addAll(e);
        return r;
    }).forEach(tryCatchStatement::addCatch);
    return configureAST(tryWithResourcesASTTransformation.transform(configureAST(tryCatchStatement, ctx)), ctx);
}

Example 4 with THIS

use of org.apache.groovy.parser.antlr4.GroovyLexer.THIS in project groovy by apache.

the class AstBuilder method visitPathElement.

@Override
public Expression visitPathElement(final PathElementContext ctx) {
    Expression baseExpr = ctx.getNodeMetaData(PATH_EXPRESSION_BASE_EXPR);
    Objects.requireNonNull(baseExpr, "baseExpr is required!");
    if (asBoolean(ctx.namePart())) {
        Expression namePartExpr = this.visitNamePart(ctx.namePart());
        GenericsType[] genericsTypes = this.visitNonWildcardTypeArguments(ctx.nonWildcardTypeArguments());
        if (asBoolean(ctx.DOT())) {
            boolean isSafeChain = this.isTrue(baseExpr, PATH_EXPRESSION_BASE_EXPR_SAFE_CHAIN);
            return this.createDotExpression(ctx, baseExpr, namePartExpr, genericsTypes, isSafeChain);
        } else if (asBoolean(ctx.SAFE_DOT())) {
            return this.createDotExpression(ctx, baseExpr, namePartExpr, genericsTypes, true);
        } else if (asBoolean(ctx.SAFE_CHAIN_DOT())) {
            // e.g. obj??.a  OR obj??.@a
            Expression expression = createDotExpression(ctx, baseExpr, namePartExpr, genericsTypes, true);
            expression.putNodeMetaData(PATH_EXPRESSION_BASE_EXPR_SAFE_CHAIN, true);
            return expression;
        } else if (asBoolean(ctx.METHOD_POINTER())) {
            // e.g. obj.&m
            return configureAST(new MethodPointerExpression(baseExpr, namePartExpr), ctx);
        } else if (asBoolean(ctx.METHOD_REFERENCE())) {
            // e.g. obj::m
            return configureAST(new MethodReferenceExpression(baseExpr, namePartExpr), ctx);
        } else if (asBoolean(ctx.SPREAD_DOT())) {
            if (asBoolean(ctx.AT())) {
                // e.g. obj*.@a
                AttributeExpression attributeExpression = new AttributeExpression(baseExpr, namePartExpr, true);
                attributeExpression.setSpreadSafe(true);
                return configureAST(attributeExpression, ctx);
            } else {
                // e.g. obj*.p
                PropertyExpression propertyExpression = new PropertyExpression(baseExpr, namePartExpr, true);
                propertyExpression.putNodeMetaData(PATH_EXPRESSION_BASE_EXPR_GENERICS_TYPES, genericsTypes);
                propertyExpression.setSpreadSafe(true);
                return configureAST(propertyExpression, ctx);
            }
        }
    } else if (asBoolean(ctx.creator())) {
        CreatorContext creatorContext = ctx.creator();
        creatorContext.putNodeMetaData(ENCLOSING_INSTANCE_EXPRESSION, baseExpr);
        return configureAST(this.visitCreator(creatorContext), ctx);
    } else if (asBoolean(ctx.indexPropertyArgs())) {
        // e.g. list[1, 3, 5]
        Tuple2<Token, Expression> tuple = this.visitIndexPropertyArgs(ctx.indexPropertyArgs());
        boolean isSafeChain = this.isTrue(baseExpr, PATH_EXPRESSION_BASE_EXPR_SAFE_CHAIN);
        return configureAST(new BinaryExpression(baseExpr, createGroovyToken(tuple.getV1()), tuple.getV2(), isSafeChain || asBoolean(ctx.indexPropertyArgs().SAFE_INDEX())), ctx);
    } else if (asBoolean(ctx.namedPropertyArgs())) {
        // this is a special way to signify a cast, e.g. Person[name: 'Daniel.Sun', location: 'Shanghai']
        List<MapEntryExpression> mapEntryExpressionList = this.visitNamedPropertyArgs(ctx.namedPropertyArgs());
        Expression right;
        Expression firstKeyExpression;
        int mapEntryExpressionListSize = mapEntryExpressionList.size();
        if (mapEntryExpressionListSize == 0) {
            // expecting list of MapEntryExpressions later so use SpreadMap to smuggle empty MapExpression to later stages
            right = configureAST(new SpreadMapExpression(configureAST(new MapExpression(), ctx.namedPropertyArgs())), ctx.namedPropertyArgs());
        } else if (mapEntryExpressionListSize == 1 && (firstKeyExpression = mapEntryExpressionList.get(0).getKeyExpression()) instanceof SpreadMapExpression) {
            right = firstKeyExpression;
        } else {
            ListExpression listExpression = configureAST(new ListExpression(mapEntryExpressionList.stream().map(e -> {
                if (e.getKeyExpression() instanceof SpreadMapExpression) {
                    return e.getKeyExpression();
                }
                return e;
            }).collect(Collectors.toList())), ctx.namedPropertyArgs());
            listExpression.setWrapped(true);
            right = listExpression;
        }
        NamedPropertyArgsContext namedPropertyArgsContext = ctx.namedPropertyArgs();
        Token token = (namedPropertyArgsContext.LBRACK() == null ? namedPropertyArgsContext.SAFE_INDEX() : namedPropertyArgsContext.LBRACK()).getSymbol();
        return configureAST(new BinaryExpression(baseExpr, createGroovyToken(token), right), ctx);
    } else if (asBoolean(ctx.arguments())) {
        Expression argumentsExpr = this.visitArguments(ctx.arguments());
        configureAST(argumentsExpr, ctx);
        if (isInsideParentheses(baseExpr)) {
            // e.g. (obj.x)(), (obj.@x)()
            return configureAST(createCallMethodCallExpression(baseExpr, argumentsExpr), ctx);
        }
        if (baseExpr instanceof AttributeExpression) {
            // e.g. obj.@a(1, 2)
            AttributeExpression attributeExpression = (AttributeExpression) baseExpr;
            // whether attributeExpression is spread safe or not, we must reset it as false
            attributeExpression.setSpreadSafe(false);
            return configureAST(createCallMethodCallExpression(attributeExpression, argumentsExpr, true), ctx);
        }
        if (baseExpr instanceof PropertyExpression) {
            // e.g. obj.a(1, 2)
            MethodCallExpression methodCallExpression = this.createMethodCallExpression((PropertyExpression) baseExpr, argumentsExpr);
            return configureAST(methodCallExpression, ctx);
        }
        if (baseExpr instanceof VariableExpression) {
            // void and primitive type AST node must be an instance of VariableExpression
            String baseExprText = baseExpr.getText();
            if (VOID_STR.equals(baseExprText)) {
                // e.g. void()
                return configureAST(this.createCallMethodCallExpression(this.createConstantExpression(baseExpr), argumentsExpr), ctx);
            } else if (isPrimitiveType(baseExprText)) {
                // e.g. int(), long(), float(), etc.
                throw this.createParsingFailedException("Primitive type literal: " + baseExprText + " cannot be used as a method name", ctx);
            }
        }
        if (// e.g. m()
        baseExpr instanceof VariableExpression || // e.g. "$m"()
        baseExpr instanceof GStringExpression || (baseExpr instanceof ConstantExpression && this.isTrue(baseExpr, IS_STRING))) {
            // e.g. "m"()
            String baseExprText = baseExpr.getText();
            if (THIS_STR.equals(baseExprText) || SUPER_STR.equals(baseExprText)) {
                // @MapConstructor(pre={ super(args?.first, args?.last); args = args ?: [:] }, post = { first = first?.toUpperCase() })
                if (visitingClosureCount > 0) {
                    return configureAST(new MethodCallExpression(baseExpr, baseExprText, argumentsExpr), ctx);
                }
                return configureAST(new ConstructorCallExpression(SUPER_STR.equals(baseExprText) ? ClassNode.SUPER : ClassNode.THIS, argumentsExpr), ctx);
            }
            MethodCallExpression methodCallExpression = this.createMethodCallExpression(baseExpr, argumentsExpr);
            return configureAST(methodCallExpression, ctx);
        }
        // e.g. 1(), 1.1(), ((int) 1 / 2)(1, 2), {a, b -> a + b }(1, 2), m()()
        return configureAST(this.createCallMethodCallExpression(baseExpr, argumentsExpr), ctx);
    } else if (asBoolean(ctx.closureOrLambdaExpression())) {
        ClosureExpression closureExpression = this.visitClosureOrLambdaExpression(ctx.closureOrLambdaExpression());
        if (baseExpr instanceof MethodCallExpression) {
            MethodCallExpression methodCallExpression = (MethodCallExpression) baseExpr;
            Expression argumentsExpression = methodCallExpression.getArguments();
            if (argumentsExpression instanceof ArgumentListExpression) {
                // normal arguments, e.g. 1, 2
                ArgumentListExpression argumentListExpression = (ArgumentListExpression) argumentsExpression;
                argumentListExpression.getExpressions().add(closureExpression);
                return configureAST(methodCallExpression, ctx);
            }
            if (argumentsExpression instanceof TupleExpression) {
                // named arguments, e.g. x: 1, y: 2
                TupleExpression tupleExpression = (TupleExpression) argumentsExpression;
                NamedArgumentListExpression namedArgumentListExpression = (NamedArgumentListExpression) tupleExpression.getExpression(0);
                if (asBoolean(tupleExpression.getExpressions())) {
                    methodCallExpression.setArguments(configureAST(new ArgumentListExpression(configureAST(new MapExpression(namedArgumentListExpression.getMapEntryExpressions()), namedArgumentListExpression), closureExpression), tupleExpression));
                } else {
                    // the branch should never reach, because named arguments must not be empty
                    methodCallExpression.setArguments(configureAST(new ArgumentListExpression(closureExpression), tupleExpression));
                }
                return configureAST(methodCallExpression, ctx);
            }
        }
        if (baseExpr instanceof PropertyExpression) {
            // e.g. obj.m { }
            MethodCallExpression methodCallExpression = this.createMethodCallExpression((PropertyExpression) baseExpr, configureAST(new ArgumentListExpression(closureExpression), closureExpression));
            return configureAST(methodCallExpression, ctx);
        }
        if (// e.g. m { }
        baseExpr instanceof VariableExpression || // e.g. "$m" { }
        baseExpr instanceof GStringExpression || (baseExpr instanceof ConstantExpression && this.isTrue(baseExpr, IS_STRING))) {
            // e.g. "m" { }
            MethodCallExpression methodCallExpression = this.createMethodCallExpression(baseExpr, configureAST(new ArgumentListExpression(closureExpression), closureExpression));
            return configureAST(methodCallExpression, ctx);
        }
        // e.g. 1 { }, 1.1 { }, (1 / 2) { }, m() { }, { -> ... } { }
        MethodCallExpression methodCallExpression = this.createCallMethodCallExpression(baseExpr, configureAST(new ArgumentListExpression(closureExpression), closureExpression));
        return configureAST(methodCallExpression, ctx);
    }
    throw createParsingFailedException("Unsupported path element: " + ctx.getText(), ctx);
}

Example 5 with THIS

use of org.apache.groovy.parser.antlr4.GroovyLexer.THIS in project groovy by apache.

the class LexerFrame method scanScript.

private void scanScript(final StringReader reader) throws Exception {
    scriptPane.read(reader, null);
    reader.reset();
    // create lexer
    final GroovyLangLexer lexer = new GroovyLangLexer(reader);
    tokenPane.setEditable(true);
    tokenPane.setText("");
    int line = 1;
    final ButtonGroup bg = new ButtonGroup();
    Token token;
    int tokenCnt = 0;
    while (true) {
        token = lexer.nextToken();
        JToggleButton tokenButton = new JToggleButton(tokens.get(token.getType()));
        tokenButton.setFont(MONOSPACED_FONT);
        bg.add(tokenButton);
        tokenButton.addActionListener(this);
        tokenButton.setToolTipText(token.getText());
        tokenButton.putClientProperty("token", token);
        tokenButton.setMargin(new Insets(0, 1, 0, 1));
        tokenButton.setFocusPainted(false);
        if (token.getLine() > line) {
            tokenPane.getDocument().insertString(tokenPane.getDocument().getLength(), "\n", null);
            line = token.getLine();
        }
        insertComponent(tokenButton);
        tokenCnt++;
        if (eof().equals(token.getType())) {
            break;
        }
    }
    tokenStreamLabel.setText(" Token Stream(" + tokenCnt + "):");
    tokenPane.setEditable(false);
    tokenPane.setCaretPosition(0);
    reader.close();
}