Example 1 with Token
use of com.laytonsmith.core.constructs.Token in project CommandHelper by EngineHub.
the class Script method compileRight.
public void compileRight() throws ConfigCompileException, ConfigCompileGroupException {
List<Token> temp = new ArrayList<>();
right = new ArrayList<>();
for (Token t : fullRight) {
if (t.type == TType.SEPERATOR) {
right.add(temp);
temp = new ArrayList<>();
} else {
if (t.type == TType.WHITESPACE) {
// Whitespace is ignored on the right side
continue;
}
temp.add(t);
}
}
right.add(temp);
cright = new ArrayList<>();
for (List<Token> l : right) {
cright.add(MethodScriptCompiler.compile(new TokenStream(l, fileOptions)));
}
}
Also used :
TokenStream(com.laytonsmith.core.compiler.TokenStream)
ArrayList(java.util.ArrayList)
Token(com.laytonsmith.core.constructs.Token)
Example 2 with Token
use of com.laytonsmith.core.constructs.Token in project CommandHelper by EngineHub.
the class MethodScriptCompiler method compile.
/**
* Compiles the token stream into a valid ParseTree. This also includes optimization and reduction.
*
* @param stream The token stream, as generated by {@link #lex(String, File, boolean) lex}
* @return A fully compiled, optimized, and reduced parse tree. If {@code stream} is null or empty, null is
* returned.
* @throws ConfigCompileException If the script contains syntax errors. Additionally, during optimization, certain
* methods may cause compile errors. Any function that can optimize static occurrences and throws a
* {@link ConfigRuntimeException} will have that exception converted to a ConfigCompileException.
*/
public static ParseTree compile(TokenStream stream) throws ConfigCompileException, ConfigCompileGroupException {
Set<ConfigCompileException> compilerErrors = new HashSet<>();
if (stream == null || stream.isEmpty()) {
return null;
}
Target unknown;
try {
// Instead of using Target.UNKNOWN, we can at least set the file.
unknown = new Target(0, stream.get(0).target.file(), 0);
} catch (Exception e) {
unknown = Target.UNKNOWN;
}
// Remove all newlines and whitespaces.
ListIterator<Token> it = stream.listIterator(0);
while (it.hasNext()) {
if (it.next().type.isWhitespace()) {
it.remove();
}
}
// Get the file options.
final FileOptions fileOptions = stream.getFileOptions();
ParseTree tree = new ParseTree(fileOptions);
tree.setData(CNull.NULL);
Stack<ParseTree> parents = new Stack<>();
/**
* constructCount is used to determine if we need to use autoconcat when reaching a FUNC_END. The previous
* constructs, if the count is greater than 1, will be moved down into an autoconcat.
*/
Stack<AtomicInteger> constructCount = new Stack<>();
constructCount.push(new AtomicInteger(0));
parents.push(tree);
tree.addChild(new ParseTree(new CFunction("__autoconcat__", unknown), fileOptions));
parents.push(tree.getChildAt(0));
tree = tree.getChildAt(0);
constructCount.push(new AtomicInteger(0));
/**
* The array stack is used to keep track of the number of square braces in use.
*/
Stack<AtomicInteger> arrayStack = new Stack<>();
arrayStack.add(new AtomicInteger(-1));
Stack<AtomicInteger> minusArrayStack = new Stack<>();
Stack<AtomicInteger> minusFuncStack = new Stack<>();
int parens = 0;
Token t = null;
int bracketCount = 0;
// Create a Token array to iterate over, rather than using the LinkedList's O(n) get() method.
Token[] tokenArray = stream.toArray(new Token[stream.size()]);
for (int i = 0; i < tokenArray.length; i++) {
t = tokenArray[i];
Token prev1 = i - 1 >= 0 ? tokenArray[i - 1] : new Token(TType.UNKNOWN, "", t.target);
Token next1 = i + 1 < stream.size() ? tokenArray[i + 1] : new Token(TType.UNKNOWN, "", t.target);
Token next2 = i + 2 < stream.size() ? tokenArray[i + 2] : new Token(TType.UNKNOWN, "", t.target);
Token next3 = i + 3 < stream.size() ? tokenArray[i + 3] : new Token(TType.UNKNOWN, "", t.target);
// Brace handling
if (t.type == TType.LCURLY_BRACKET) {
ParseTree b = new ParseTree(new CFunction("__cbrace__", t.getTarget()), fileOptions);
tree.addChild(b);
tree = b;
parents.push(b);
bracketCount++;
constructCount.push(new AtomicInteger(0));
continue;
}
if (t.type == TType.RCURLY_BRACKET) {
bracketCount--;
if (constructCount.peek().get() > 1) {
// We need to autoconcat some stuff
int stacks = constructCount.peek().get();
int replaceAt = tree.getChildren().size() - stacks;
ParseTree c = new ParseTree(new CFunction("__autoconcat__", tree.getTarget()), fileOptions);
List<ParseTree> subChildren = new ArrayList<>();
for (int b = replaceAt; b < tree.numberOfChildren(); b++) {
subChildren.add(tree.getChildAt(b));
}
c.setChildren(subChildren);
if (replaceAt > 0) {
List<ParseTree> firstChildren = new ArrayList<>();
for (int d = 0; d < replaceAt; d++) {
firstChildren.add(tree.getChildAt(d));
}
tree.setChildren(firstChildren);
} else {
tree.removeChildren();
}
tree.addChild(c);
}
parents.pop();
tree = parents.peek();
constructCount.pop();
try {
constructCount.peek().incrementAndGet();
} catch (EmptyStackException e) {
throw new ConfigCompileException("Unexpected end curly brace", t.target);
}
continue;
}
// Associative array/label handling
if (t.type == TType.LABEL && tree.getChildren().size() > 0) {
// If it's not an atomic identifier it's an error.
if (!prev1.type.isAtomicLit() && prev1.type != TType.IVARIABLE && prev1.type != TType.KEYWORD) {
ConfigCompileException error = new ConfigCompileException("Invalid label specified", t.getTarget());
if (prev1.type == TType.FUNC_END) {
// and stop compilation at this point.
throw error;
}
compilerErrors.add(error);
}
// Wrap previous construct in a CLabel
ParseTree cc = tree.getChildren().get(tree.getChildren().size() - 1);
tree.removeChildAt(tree.getChildren().size() - 1);
tree.addChild(new ParseTree(new CLabel(cc.getData()), fileOptions));
continue;
}
// Array notation handling
if (t.type.equals(TType.LSQUARE_BRACKET)) {
arrayStack.push(new AtomicInteger(tree.getChildren().size() - 1));
continue;
} else if (t.type.equals(TType.RSQUARE_BRACKET)) {
boolean emptyArray = false;
if (prev1.type.equals(TType.LSQUARE_BRACKET)) {
emptyArray = true;
}
if (arrayStack.size() == 1) {
throw new ConfigCompileException("Mismatched square bracket", t.target);
}
// array is the location of the array
int array = arrayStack.pop().get();
// index is the location of the first node with the index
int index = array + 1;
if (!tree.hasChildren() || array == -1) {
throw new ConfigCompileException("Brackets are illegal here", t.target);
}
ParseTree myArray = tree.getChildAt(array);
ParseTree myIndex;
if (!emptyArray) {
myIndex = new ParseTree(new CFunction("__autoconcat__", myArray.getTarget()), fileOptions);
for (int j = index; j < tree.numberOfChildren(); j++) {
myIndex.addChild(tree.getChildAt(j));
}
} else {
myIndex = new ParseTree(new CSlice("0..-1", t.target), fileOptions);
}
tree.setChildren(tree.getChildren().subList(0, array));
ParseTree arrayGet = new ParseTree(new CFunction("array_get", t.target), fileOptions);
arrayGet.addChild(myArray);
arrayGet.addChild(myIndex);
// Check if the @var[...] had a negating "-" in front. If so, add a neg().
if (!minusArrayStack.isEmpty() && arrayStack.size() + 1 == minusArrayStack.peek().get()) {
if (!next1.type.equals(TType.LSQUARE_BRACKET)) {
// Wait if there are more array_get's comming.
ParseTree negTree = new ParseTree(new CFunction("neg", unknown), fileOptions);
negTree.addChild(arrayGet);
tree.addChild(negTree);
minusArrayStack.pop();
} else {
// Negate the next array_get instead, so just add this one to the tree.
tree.addChild(arrayGet);
}
} else {
tree.addChild(arrayGet);
}
constructCount.peek().set(constructCount.peek().get() - myIndex.numberOfChildren());
continue;
}
// Smart strings
if (t.type == TType.SMART_STRING) {
if (t.val().contains("@")) {
ParseTree function = new ParseTree(fileOptions);
function.setData(new CFunction(new Compiler.smart_string().getName(), t.target));
ParseTree string = new ParseTree(fileOptions);
string.setData(new CString(t.value, t.target));
function.addChild(string);
tree.addChild(function);
} else {
tree.addChild(new ParseTree(new CString(t.val(), t.target), fileOptions));
}
constructCount.peek().incrementAndGet();
continue;
}
if (t.type == TType.DEREFERENCE) {
// Currently unimplemented, but going ahead and making it strict
compilerErrors.add(new ConfigCompileException("The '" + t.val() + "' symbol is not currently allowed in raw strings. You must quote all" + " symbols.", t.target));
}
if (t.type.equals(TType.FUNC_NAME)) {
CFunction func = new CFunction(t.val(), t.target);
ParseTree f = new ParseTree(func, fileOptions);
tree.addChild(f);
constructCount.push(new AtomicInteger(0));
tree = f;
parents.push(f);
} else if (t.type.equals(TType.FUNC_START)) {
if (!prev1.type.equals(TType.FUNC_NAME)) {
throw new ConfigCompileException("Unexpected parenthesis", t.target);
}
parens++;
} else if (t.type.equals(TType.FUNC_END)) {
if (parens <= 0) {
throw new ConfigCompileException("Unexpected parenthesis", t.target);
}
parens--;
// Pop function.
parents.pop();
if (constructCount.peek().get() > 1) {
// We need to autoconcat some stuff
int stacks = constructCount.peek().get();
int replaceAt = tree.getChildren().size() - stacks;
ParseTree c = new ParseTree(new CFunction("__autoconcat__", tree.getTarget()), fileOptions);
List<ParseTree> subChildren = new ArrayList<>();
for (int b = replaceAt; b < tree.numberOfChildren(); b++) {
subChildren.add(tree.getChildAt(b));
}
c.setChildren(subChildren);
if (replaceAt > 0) {
List<ParseTree> firstChildren = new ArrayList<>();
for (int d = 0; d < replaceAt; d++) {
firstChildren.add(tree.getChildAt(d));
}
tree.setChildren(firstChildren);
} else {
tree.removeChildren();
}
tree.addChild(c);
}
constructCount.pop();
try {
constructCount.peek().incrementAndGet();
} catch (EmptyStackException e) {
throw new ConfigCompileException("Unexpected end parenthesis", t.target);
}
try {
tree = parents.peek();
} catch (EmptyStackException e) {
throw new ConfigCompileException("Unexpected end parenthesis", t.target);
}
// Handle "-func(args)" and "-func(args)[index]".
if (!minusFuncStack.isEmpty() && minusFuncStack.peek().get() == parens + 1) {
if (next1.type.equals(TType.LSQUARE_BRACKET)) {
// Move the negation to the array_get which contains this function.
// +1 because the bracket isn't counted yet.
minusArrayStack.push(new AtomicInteger(arrayStack.size() + 1));
} else {
// Negate this function.
ParseTree negTree = new ParseTree(new CFunction("neg", unknown), fileOptions);
negTree.addChild(tree.getChildAt(tree.numberOfChildren() - 1));
tree.removeChildAt(tree.numberOfChildren() - 1);
tree.addChildAt(tree.numberOfChildren(), negTree);
}
minusFuncStack.pop();
}
} else if (t.type.equals(TType.COMMA)) {
if (constructCount.peek().get() > 1) {
int stacks = constructCount.peek().get();
int replaceAt = tree.getChildren().size() - stacks;
ParseTree c = new ParseTree(new CFunction("__autoconcat__", unknown), fileOptions);
List<ParseTree> subChildren = new ArrayList<>();
for (int b = replaceAt; b < tree.numberOfChildren(); b++) {
subChildren.add(tree.getChildAt(b));
}
c.setChildren(subChildren);
if (replaceAt > 0) {
List<ParseTree> firstChildren = new ArrayList<>();
for (int d = 0; d < replaceAt; d++) {
firstChildren.add(tree.getChildAt(d));
}
tree.setChildren(firstChildren);
} else {
tree.removeChildren();
}
tree.addChild(c);
}
constructCount.peek().set(0);
continue;
}
if (t.type == TType.SLICE) {
// "empty first" slice notation. Compare this to the code below.
try {
CSlice slice;
String value = next1.val();
if (next1.type == TType.MINUS || next1.type == TType.PLUS) {
value = next1.val() + next2.val();
i++;
}
slice = new CSlice(".." + value, t.getTarget());
i++;
tree.addChild(new ParseTree(slice, fileOptions));
constructCount.peek().incrementAndGet();
continue;
} catch (ConfigRuntimeException ex) {
// turn them into a CCE.
throw new ConfigCompileException(ex);
}
}
if (next1.type.equals(TType.SLICE)) {
// Slice notation handling
try {
CSlice slice;
if (t.type.isSeparator() || (t.type.isWhitespace() && prev1.type.isSeparator()) || t.type.isKeyword()) {
// empty first
String value = next2.val();
i++;
if (next2.type == TType.MINUS || next2.type == TType.PLUS) {
value = next2.val() + next3.val();
i++;
}
slice = new CSlice(".." + value, next1.getTarget());
if (t.type.isKeyword()) {
tree.addChild(new ParseTree(new CKeyword(t.val(), t.getTarget()), fileOptions));
constructCount.peek().incrementAndGet();
}
} else if (next2.type.isSeparator() || next2.type.isKeyword()) {
// empty last
String modifier = "";
if (prev1.type == TType.MINUS || prev1.type == TType.PLUS) {
// The negative would have already been inserted into the tree
modifier = prev1.val();
tree.removeChildAt(tree.getChildren().size() - 1);
}
slice = new CSlice(modifier + t.value + "..", t.target);
} else {
// both are provided
String modifier1 = "";
if (prev1.type == TType.MINUS || prev1.type == TType.PLUS) {
// It's a negative, incorporate that here, and remove the
// minus from the tree
modifier1 = prev1.val();
tree.removeChildAt(tree.getChildren().size() - 1);
}
Token first = t;
if (first.type.isWhitespace()) {
first = prev1;
}
Token second = next2;
i++;
String modifier2 = "";
if (next2.type == TType.MINUS || next2.type == TType.PLUS) {
modifier2 = next2.val();
second = next3;
i++;
}
slice = new CSlice(modifier1 + first.value + ".." + modifier2 + second.value, t.target);
}
i++;
tree.addChild(new ParseTree(slice, fileOptions));
constructCount.peek().incrementAndGet();
continue;
} catch (ConfigRuntimeException ex) {
// turn them into a CCE.
throw new ConfigCompileException(ex);
}
} else if (t.type == TType.LIT) {
Construct c = Static.resolveConstruct(t.val(), t.target);
if (c instanceof CString && fileOptions.isStrict()) {
compilerErrors.add(new ConfigCompileException("Bare strings are not allowed in strict mode", t.target));
} else if ((c instanceof CInt || c instanceof CDecimal) && next1.type == TType.DOT && next2.type == TType.LIT) {
// minus zero before decimals and leading zeroes after decimals
try {
if (t.value.startsWith("0m")) {
// CDecimal
String neg = "";
if (prev1.value.equals("-")) {
neg = "-";
}
c = new CDecimal(neg + t.value.substring(2) + '.' + next2.value, t.target);
} else {
// CDouble
c = new CDouble(Double.parseDouble(t.val() + '.' + next2.val()), t.target);
}
i += 2;
} catch (NumberFormatException e) {
// Not a double
}
}
tree.addChild(new ParseTree(c, fileOptions));
constructCount.peek().incrementAndGet();
} else if (t.type.equals(TType.STRING) || t.type.equals(TType.COMMAND)) {
tree.addChild(new ParseTree(new CString(t.val(), t.target), fileOptions));
constructCount.peek().incrementAndGet();
} else if (t.type.equals(TType.IDENTIFIER)) {
tree.addChild(new ParseTree(new CPreIdentifier(t.val(), t.target), fileOptions));
constructCount.peek().incrementAndGet();
} else if (t.type.isKeyword()) {
tree.addChild(new ParseTree(new CKeyword(t.val(), t.getTarget()), fileOptions));
constructCount.peek().incrementAndGet();
} else if (t.type.equals(TType.IVARIABLE)) {
tree.addChild(new ParseTree(new IVariable(t.val(), t.target), fileOptions));
constructCount.peek().incrementAndGet();
} else if (t.type.equals(TType.UNKNOWN)) {
tree.addChild(new ParseTree(Static.resolveConstruct(t.val(), t.target), fileOptions));
constructCount.peek().incrementAndGet();
} else if (t.type.isSymbol()) {
// Also handles "-function()" and "-@var[index]".
if (t.type.equals(TType.MINUS) && !prev1.type.isAtomicLit() && !prev1.type.equals(TType.IVARIABLE) && !prev1.type.equals(TType.VARIABLE) && !prev1.type.equals(TType.RCURLY_BRACKET) && !prev1.type.equals(TType.RSQUARE_BRACKET) && !prev1.type.equals(TType.FUNC_END) && (next1.type.equals(TType.IVARIABLE) || next1.type.equals(TType.VARIABLE) || next1.type.equals(TType.FUNC_NAME))) {
// Check if we are negating a value from an array, function or variable.
if (next2.type.equals(TType.LSQUARE_BRACKET)) {
// +1 because the bracket isn't counted yet.
minusArrayStack.push(new AtomicInteger(arrayStack.size() + 1));
} else if (next1.type.equals(TType.FUNC_NAME)) {
// +1 because the function isn't counted yet.
minusFuncStack.push(new AtomicInteger(parens + 1));
} else {
ParseTree negTree = new ParseTree(new CFunction("neg", unknown), fileOptions);
negTree.addChild(new ParseTree(new IVariable(next1.value, next1.target), fileOptions));
tree.addChild(negTree);
constructCount.peek().incrementAndGet();
// Skip the next variable as we've just handled it.
i++;
}
} else {
tree.addChild(new ParseTree(new CSymbol(t.val(), t.type, t.target), fileOptions));
constructCount.peek().incrementAndGet();
}
} else if (t.type == TType.DOT) {
// Check for doubles that start with a decimal, otherwise concat
Construct c = null;
if (next1.type == TType.LIT && prev1.type != TType.STRING && prev1.type != TType.SMART_STRING) {
try {
c = new CDouble(Double.parseDouble('.' + next1.val()), t.target);
i++;
} catch (NumberFormatException e) {
// Not a double
}
}
if (c == null) {
c = new CSymbol(".", TType.CONCAT, t.target);
}
tree.addChild(new ParseTree(c, fileOptions));
constructCount.peek().incrementAndGet();
} else if (t.type.equals(TType.VARIABLE) || t.type.equals(TType.FINAL_VAR)) {
tree.addChild(new ParseTree(new Variable(t.val(), null, false, t.type.equals(TType.FINAL_VAR), t.target), fileOptions));
constructCount.peek().incrementAndGet();
// right_vars.add(new Variable(t.val(), null, t.line_num));
}
}
assert t != null;
if (arrayStack.size() != 1) {
throw new ConfigCompileException("Mismatched square brackets", t.target);
}
if (parens != 0) {
throw new ConfigCompileException("Mismatched parenthesis", t.target);
}
if (bracketCount != 0) {
throw new ConfigCompileException("Mismatched curly braces", t.target);
}
Stack<List<Procedure>> procs = new Stack<>();
procs.add(new ArrayList<Procedure>());
processKeywords(tree);
optimizeAutoconcats(tree, compilerErrors);
optimize(tree, procs, compilerErrors);
link(tree, compilerErrors);
checkLabels(tree, compilerErrors);
checkBreaks(tree, compilerErrors);
if (!compilerErrors.isEmpty()) {
if (compilerErrors.size() == 1) {
// Just throw the one CCE
for (ConfigCompileException e : compilerErrors) {
throw e;
}
} else {
throw new ConfigCompileGroupException(compilerErrors);
}
}
parents.pop();
tree = parents.pop();
return tree;
}
Also used :
CLabel(com.laytonsmith.core.constructs.CLabel)
IVariable(com.laytonsmith.core.constructs.IVariable)
Variable(com.laytonsmith.core.constructs.Variable)
CSymbol(com.laytonsmith.core.constructs.CSymbol)
CPreIdentifier(com.laytonsmith.core.constructs.CPreIdentifier)
IVariable(com.laytonsmith.core.constructs.IVariable)
ArrayList(java.util.ArrayList)
Token(com.laytonsmith.core.constructs.Token)
CString(com.laytonsmith.core.constructs.CString)
ConfigRuntimeException(com.laytonsmith.core.exceptions.ConfigRuntimeException)
ConfigCompileException(com.laytonsmith.core.exceptions.ConfigCompileException)
CString(com.laytonsmith.core.constructs.CString)
EmptyStackException(java.util.EmptyStackException)
Target(com.laytonsmith.core.constructs.Target)
CSlice(com.laytonsmith.core.constructs.CSlice)
KeywordList(com.laytonsmith.core.compiler.KeywordList)
FunctionList(com.laytonsmith.core.functions.FunctionList)
List(java.util.List)
ArrayList(java.util.ArrayList)
LinkedList(java.util.LinkedList)
CDecimal(com.laytonsmith.core.constructs.CDecimal)
HashSet(java.util.HashSet)
Compiler(com.laytonsmith.core.functions.Compiler)
CFunction(com.laytonsmith.core.constructs.CFunction)
CDouble(com.laytonsmith.core.constructs.CDouble)
URISyntaxException(java.net.URISyntaxException)
ProgramFlowManipulationException(com.laytonsmith.core.exceptions.ProgramFlowManipulationException)
ConfigCompileException(com.laytonsmith.core.exceptions.ConfigCompileException)
DataSourceException(com.laytonsmith.persistence.DataSourceException)
NoSuchElementException(java.util.NoSuchElementException)
EmptyStackException(java.util.EmptyStackException)
IOException(java.io.IOException)
ConfigRuntimeException(com.laytonsmith.core.exceptions.ConfigRuntimeException)
ConfigCompileGroupException(com.laytonsmith.core.exceptions.ConfigCompileGroupException)
Stack(java.util.Stack)
CInt(com.laytonsmith.core.constructs.CInt)
AtomicInteger(java.util.concurrent.atomic.AtomicInteger)
Construct(com.laytonsmith.core.constructs.Construct)
CKeyword(com.laytonsmith.core.constructs.CKeyword)
ConfigCompileGroupException(com.laytonsmith.core.exceptions.ConfigCompileGroupException)
FileOptions(com.laytonsmith.core.compiler.FileOptions)
Example 3 with Token
use of com.laytonsmith.core.constructs.Token in project CommandHelper by EngineHub.
the class MethodScriptCompiler method lex.
/**
* Lexes the script, and turns it into a token stream. This looks through the script character by character.
*
* @param script The script to lex
* @param file The file this script came from, or potentially null if the code is from a dynamic source
* @param inPureMScript If the script is in pure MethodScript, this should be true. Pure MethodScript is defined as
* code that doesn't have command alias wrappers.
* @param saveAllTokens If this script is planning to be compiled, then this value should always be false, however,
* if the calling code needs all tokens for informational purposes (and doesn't plan on actually compiling the code)
* then this can be true. If true, all tokens are saved, including comments and (some) whitespace. Given this lexing
* stream, the exact source code could be re-constructed.
*
* A note on whitespace: The whitespace tokens are not guaranteed to be accurate, however, the column information
* is. If you have two tokens t1 and t2, each with a value of length 1, where the columns are 1 and 5, then that
* means there are 4 spaces between the two.
* @return A stream of tokens
* @throws ConfigCompileException If compilation fails due to bad syntax
*/
public static TokenStream lex(String script, File file, boolean inPureMScript, boolean saveAllTokens) throws ConfigCompileException {
if (script.isEmpty()) {
return new TokenStream(new LinkedList<>(), "");
}
if ((int) script.charAt(0) == 65279) {
// Remove the UTF-8 Byte Order Mark, if present.
script = script.substring(1);
}
final StringBuilder fileOptions = new StringBuilder();
script = script.replaceAll("\r\n", "\n");
script = script + "\n";
final Set<String> keywords = KeywordList.getKeywordNames();
final TokenStream token_list = new TokenStream();
// Set our state variables.
boolean state_in_quote = false;
int quoteLineNumberStart = 1;
boolean in_smart_quote = false;
int smartQuoteLineNumberStart = 1;
boolean in_comment = false;
int commentLineNumberStart = 1;
boolean comment_is_block = false;
boolean in_opt_var = false;
boolean inCommand = (!inPureMScript);
boolean inMultiline = false;
boolean in_smart_comment = false;
boolean in_file_options = false;
int fileOptionsLineNumberStart = 1;
StringBuilder buf = new StringBuilder();
int line_num = 1;
int column = 1;
int lastColumn = 0;
Target target = Target.UNKNOWN;
// Lex the script character by character.
for (int i = 0; i < script.length(); i++) {
Character c = script.charAt(i);
Character c2 = null;
if (i < script.length() - 1) {
c2 = script.charAt(i + 1);
}
column += i - lastColumn;
lastColumn = i;
if (c == '\n') {
line_num++;
column = 1;
if (!inMultiline && !inPureMScript) {
inCommand = true;
}
}
if (buf.length() == 0) {
target = new Target(line_num, file, column);
}
// If we are in file options, add the character to the buffer if it's not a file options end character.
if (in_file_options) {
// If support for more escaped characters would be desired in the future, it could be added here.
switch(c) {
case '\\':
{
if (c2 == '>') {
// "\>".
fileOptions.append('>');
i++;
continue;
}
}
case '>':
{
if (saveAllTokens) {
token_list.add(new Token(TType.FILE_OPTIONS_STRING, fileOptions.toString(), target));
token_list.add(new Token(TType.FILE_OPTIONS_END, ">", target));
}
in_file_options = false;
continue;
}
}
fileOptions.append(c);
continue;
}
// Comment handling. This is bypassed if we are in a string.
if (!state_in_quote && !in_smart_quote) {
switch(c) {
// Block comments start (/* and /**) and Double slash line comment start (//).
case '/':
{
if (!in_comment) {
if (c2 == '*') {
// "/*" or "/**".
buf.append("/*");
in_comment = true;
comment_is_block = true;
if (i < script.length() - 2 && script.charAt(i + 2) == '*') {
// "/**".
in_smart_comment = true;
buf.append("*");
i++;
}
commentLineNumberStart = line_num;
i++;
continue;
} else if (c2 == '/') {
// "//".
buf.append("//");
in_comment = true;
i++;
continue;
}
}
break;
}
// Line comment start (#).
case '#':
{
if (!in_comment) {
// "#".
buf.append("#");
in_comment = true;
continue;
}
break;
}
// Block comment end (*/).
case '*':
{
if (in_comment && comment_is_block && c2 == '/') {
// "*/".
if (saveAllTokens || in_smart_comment) {
buf.append("*/");
token_list.add(new Token(in_smart_comment ? TType.SMART_COMMENT : TType.COMMENT, buf.toString(), target));
}
buf = new StringBuilder();
target = new Target(line_num, file, column);
in_comment = false;
comment_is_block = false;
in_smart_comment = false;
i++;
continue;
}
break;
}
// Line comment end (\n).
case '\n':
{
if (in_comment && !comment_is_block) {
// "\n".
in_comment = false;
if (saveAllTokens) {
token_list.add(new Token(TType.COMMENT, buf.toString(), target));
token_list.add(new Token(TType.NEWLINE, "\n", new Target(line_num + 1, file, 0)));
}
buf = new StringBuilder();
target = new Target(line_num, file, column);
continue;
}
break;
}
}
}
// If we are in a comment, add the character to the buffer.
if (in_comment) {
buf.append(c);
continue;
}
// Handle non-comment non-quoted characters.
if (!state_in_quote) {
// (, ), ;, and whitespace.
matched: {
Token token;
switch(c) {
case '+':
{
if (c2 == '=') {
// "+=".
token = new Token(TType.PLUS_ASSIGNMENT, "+=", target);
i++;
} else if (c2 == '+') {
// "++".
token = new Token(TType.INCREMENT, "++", target);
i++;
} else {
// "+".
token = new Token(TType.PLUS, "+", target);
}
break;
}
case '-':
{
if (c2 == '=') {
// "-=".
token = new Token(TType.MINUS_ASSIGNMENT, "-=", target);
i++;
} else if (c2 == '-') {
// "--".
token = new Token(TType.DECREMENT, "--", target);
i++;
} else if (c2 == '>') {
// "->".
token = new Token(TType.DEREFERENCE, "->", target);
i++;
} else {
// "-".
token = new Token(TType.MINUS, "-", target);
}
break;
}
case '*':
{
if (c2 == '=') {
// "*=".
token = new Token(TType.MULTIPLICATION_ASSIGNMENT, "*=", target);
i++;
} else if (c2 == '*') {
// "**".
token = new Token(TType.EXPONENTIAL, "**", target);
i++;
} else {
// "*".
token = new Token(TType.MULTIPLICATION, "*", target);
}
break;
}
case '/':
{
if (c2 == '=') {
// "/=".
token = new Token(TType.DIVISION_ASSIGNMENT, "/=", target);
i++;
} else {
// Protect against matching commands.
if (Character.isLetter(c2)) {
// Pretend that division didn't match.
break matched;
}
token = new Token(TType.DIVISION, "/", target);
}
break;
}
case '.':
{
if (c2 == '=') {
// ".=".
token = new Token(TType.CONCAT_ASSIGNMENT, ".=", target);
i++;
} else if (c2 == '.') {
// "..".
token = new Token(TType.SLICE, "..", target);
i++;
} else {
// ".".
token = new Token(TType.DOT, ".", target);
}
break;
}
case '%':
{
token = new Token(TType.MODULO, "%", target);
break;
}
case '>':
{
if (c2 == '=') {
// ">=".
token = new Token(TType.GTE, ">=", target);
i++;
} else if (c2 == '>' && i < script.length() - 2 && script.charAt(i + 2) == '>') {
// ">>>".
token = new Token(TType.MULTILINE_START, ">>>", target);
inMultiline = true;
i += 2;
} else {
// ">".
token = new Token(TType.GT, ">", target);
}
break;
}
case '<':
{
if (c2 == '!') {
// "<!".
if (buf.length() > 0) {
token_list.add(new Token(TType.UNKNOWN, buf.toString(), target));
buf = new StringBuilder();
target = new Target(line_num, file, column);
}
if (saveAllTokens) {
token_list.add(new Token(TType.FILE_OPTIONS_START, "<!", target));
}
in_file_options = true;
fileOptionsLineNumberStart = line_num;
i++;
continue;
} else if (c2 == '=') {
// "<=".
token = new Token(TType.LTE, "<=", target);
i++;
} else if (c2 == '<' && i < script.length() - 2 && script.charAt(i + 2) == '<') {
// "<<<".
token = new Token(TType.MULTILINE_END, "<<<", target);
inMultiline = false;
i += 2;
} else {
// "<".
token = new Token(TType.LT, "<", target);
}
break;
}
case '=':
{
if (c2 == '=') {
if (i < script.length() - 2 && script.charAt(i + 2) == '=') {
// "===".
token = new Token(TType.STRICT_EQUALS, "===", target);
i += 2;
} else {
// "==".
token = new Token(TType.EQUALS, "==", target);
i++;
}
} else {
// "=".
if (inCommand) {
if (in_opt_var) {
token = new Token(TType.OPT_VAR_ASSIGN, "=", target);
} else {
token = new Token(TType.ALIAS_END, "=", target);
inCommand = false;
}
} else {
token = new Token(TType.ASSIGNMENT, "=", target);
}
}
break;
}
case '!':
{
if (c2 == '=') {
if (i < script.length() - 2 && script.charAt(i + 2) == '=') {
// "!==".
token = new Token(TType.STRICT_NOT_EQUALS, "!==", target);
i += 2;
} else {
// "!=".
token = new Token(TType.NOT_EQUALS, "!=", target);
i++;
}
} else {
// "!".
token = new Token(TType.LOGICAL_NOT, "!", target);
}
break;
}
case '&':
{
if (c2 == '&') {
if (i < script.length() - 2 && script.charAt(i + 2) == '&') {
// "&&&".
token = new Token(TType.DEFAULT_AND, "&&&", target);
i += 2;
} else {
// "&&".
token = new Token(TType.LOGICAL_AND, "&&", target);
i++;
}
} else {
// Pretend that bitwise AND didn't match.
break matched;
// token = new Token(TType.BIT_AND, "&", target);
}
break;
}
case '|':
{
if (c2 == '|') {
if (i < script.length() - 2 && script.charAt(i + 2) == '|') {
// "|||".
token = new Token(TType.DEFAULT_OR, "|||", target);
i += 2;
} else {
// "||".
token = new Token(TType.LOGICAL_OR, "||", target);
i++;
}
} else {
// Pretend that bitwise OR didn't match.
break matched;
// token = new Token(TType.BIT_OR, "|", target);
}
break;
}
// }
case ':':
{
if (c2 == ':') {
// "::".
token = new Token(TType.DEREFERENCE, "::", target);
i++;
} else {
// ":".
token = new Token(TType.LABEL, ":", target);
}
break;
}
case '{':
{
token = new Token(TType.LCURLY_BRACKET, "{", target);
break;
}
case '}':
{
token = new Token(TType.RCURLY_BRACKET, "}", target);
break;
}
case '[':
{
token = new Token(TType.LSQUARE_BRACKET, "[", target);
in_opt_var = true;
break;
}
case ']':
{
token = new Token(TType.RSQUARE_BRACKET, "]", target);
in_opt_var = false;
break;
}
case ',':
{
token = new Token(TType.COMMA, ",", target);
break;
}
case ';':
{
token = new Token(TType.SEMICOLON, ";", target);
break;
}
case '(':
{
token = new Token(TType.FUNC_START, "(", target);
// Handle the buffer or previous token, with the knowledge that a FUNC_START follows.
if (buf.length() > 0) {
if (saveAllTokens) {
// true, because we know that won't be used by the compiler.
if (KeywordList.getKeywordByName(buf.toString()) != null) {
// It's a keyword.
token_list.add(new Token(TType.KEYWORD, buf.toString(), target));
} else {
// It's not a keyword, but a normal function.
token_list.add(new Token(TType.FUNC_NAME, buf.toString(), target));
}
} else {
token_list.add(new Token(TType.FUNC_NAME, buf.toString(), target));
}
buf = new StringBuilder();
target = new Target(line_num, file, column);
} else {
// function.
try {
int count = 0;
Iterator<Token> it = token_list.descendingIterator();
Token t;
while ((t = it.next()).type == TType.WHITESPACE) {
count++;
}
if (t.type == TType.UNKNOWN) {
t.type = TType.FUNC_NAME;
// Go ahead and remove the whitespace here too, they break things.
count--;
for (int a = 0; a < count; a++) {
token_list.removeLast();
}
} else {
token_list.add(new Token(TType.FUNC_NAME, "__autoconcat__", target));
}
} catch (NoSuchElementException e) {
// This is the first element on the list, so, it's another autoconcat.
token_list.add(new Token(TType.FUNC_NAME, "__autoconcat__", target));
}
}
break;
}
case ')':
{
token = new Token(TType.FUNC_END, ")", target);
break;
}
case ' ':
{
// Whitespace case #1.
token = new Token(TType.WHITESPACE, " ", target);
break;
}
case '\t':
{
// Whitespace case #2 (TAB).
token = new Token(TType.WHITESPACE, "\t", target);
break;
}
default:
{
// No match was found at this point, so continue matching below.
break matched;
}
}
// Add previous characters as UNKNOWN token.
if (buf.length() > 0) {
token_list.add(new Token(TType.UNKNOWN, buf.toString(), target));
buf = new StringBuilder();
target = new Target(line_num, file, column);
}
// Add the new token to the token list.
token_list.add(token);
// Continue lexing.
continue;
}
}
// Handle non-comment characters that might start or stop a quoted string.
switch(c) {
case '\'':
{
if (state_in_quote && !in_smart_quote) {
token_list.add(new Token(TType.STRING, buf.toString(), target));
buf = new StringBuilder();
target = new Target(line_num, file, column);
state_in_quote = false;
continue;
} else if (!state_in_quote) {
state_in_quote = true;
quoteLineNumberStart = line_num;
in_smart_quote = false;
if (buf.length() > 0) {
token_list.add(new Token(TType.UNKNOWN, buf.toString(), target));
buf = new StringBuilder();
target = new Target(line_num, file, column);
}
continue;
} else {
// We're in a smart quote.
buf.append("'");
}
break;
}
case '"':
{
if (state_in_quote && in_smart_quote) {
token_list.add(new Token(TType.SMART_STRING, buf.toString(), target));
buf = new StringBuilder();
target = new Target(line_num, file, column);
state_in_quote = false;
in_smart_quote = false;
continue;
} else if (!state_in_quote) {
state_in_quote = true;
in_smart_quote = true;
smartQuoteLineNumberStart = line_num;
if (buf.length() > 0) {
token_list.add(new Token(TType.UNKNOWN, buf.toString(), target));
buf = new StringBuilder();
target = new Target(line_num, file, column);
}
continue;
} else {
// We're in normal quotes.
buf.append('"');
}
break;
}
case '\n':
{
// Append a newline to the buffer if it's quoted.
if (state_in_quote) {
buf.append(c);
} else {
// Newline is not quoted. Move the buffer to an UNKNOWN token and add a NEWLINE token.
if (buf.length() > 0) {
token_list.add(new Token(TType.UNKNOWN, buf.toString(), target));
buf = new StringBuilder();
target = new Target(line_num, file, column);
}
token_list.add(new Token(TType.NEWLINE, "\n", target));
}
continue;
}
case '\\':
{
// Handle backslash character outside of quotes.
if (!state_in_quote) {
token_list.add(new Token(TType.SEPERATOR, "\\", target));
break;
}
// Handle an escape sign in a quote.
switch(c2) {
case '\\':
case '\'':
case '"':
buf.append(c2);
break;
case 'n':
buf.append('\n');
break;
case 'r':
buf.append('\r');
break;
case 't':
buf.append('\t');
break;
case '0':
buf.append('\0');
break;
case 'f':
buf.append('\f');
// Form feed.
break;
case 'v':
buf.append('\u000B');
// Vertical TAB.
break;
case 'a':
buf.append('\u0007');
// Alarm.
break;
case 'b':
buf.append('\u0008');
// Backspace.
break;
case 'u':
{
// Grab the next 4 characters, and check to see if they are numbers.
if (i + 5 >= script.length()) {
throw new ConfigCompileException("Unrecognized unicode escape sequence", target);
}
String unicode = script.substring(i + 2, i + 6);
int unicodeNum;
try {
unicodeNum = Integer.parseInt(unicode, 16);
} catch (NumberFormatException e) {
throw new ConfigCompileException("Unrecognized unicode escape sequence: \\u" + unicode, target);
}
buf.append(Character.toChars(unicodeNum));
i += 4;
break;
}
case 'U':
{
// Grab the next 8 characters and check to see if they are numbers.
if (i + 9 >= script.length()) {
throw new ConfigCompileException("Unrecognized unicode escape sequence", target);
}
String unicode = script.substring(i + 2, i + 10);
int unicodeNum;
try {
unicodeNum = Integer.parseInt(unicode, 16);
} catch (NumberFormatException e) {
throw new ConfigCompileException("Unrecognized unicode escape sequence: \\u" + unicode, target);
}
buf.append(Character.toChars(unicodeNum));
i += 8;
break;
}
case '@':
{
if (!in_smart_quote) {
throw new ConfigCompileException("The escape sequence \\@ is not" + " a recognized escape sequence in a non-smart string", target);
}
buf.append("\\@");
break;
}
default:
{
// Since we might expand this list later, don't let them use unescaped backslashes.
throw new ConfigCompileException("The escape sequence \\" + c2 + " is not a recognized escape sequence", target);
}
}
i++;
continue;
}
default:
{
// Disallow Non-Breaking Space Characters.
if (!state_in_quote && c == '\u00A0') /*nbsp*/
{
throw new ConfigCompileException("NBSP character in script", target);
}
// Add the characters that didn't match anything to the buffer.
buf.append(c);
continue;
}
}
}
// Handle unended file options.
if (in_file_options) {
throw new ConfigCompileException("Unended file options. You started the the file options on line " + fileOptionsLineNumberStart, target);
}
// Handle unended string literals.
if (state_in_quote) {
if (in_smart_quote) {
throw new ConfigCompileException("Unended string literal. You started the last double quote on line " + smartQuoteLineNumberStart, target);
} else {
throw new ConfigCompileException("Unended string literal. You started the last single quote on line " + quoteLineNumberStart, target);
}
}
// Handle unended comment blocks. Since a newline is added to the end of the script, line comments are ended.
if (in_comment || comment_is_block) {
throw new ConfigCompileException("Unended block comment. You started the comment on line " + commentLineNumberStart, target);
}
// Look at the tokens and get meaning from them. Also, look for improper symbol locations
// and go ahead and absorb unary +- into the token.
ListIterator<Token> it = token_list.listIterator(0);
while (it.hasNext()) {
Token t = it.next();
// Combine whitespace tokens into one.
if (t.type == TType.WHITESPACE && it.hasNext()) {
Token next;
if ((next = it.next()).type == TType.WHITESPACE) {
t.value += next.val();
// Remove 'next'.
it.remove();
} else {
// Select 'next' <--.
it.previous();
}
// Select 't' <--.
it.previous();
// Select 't' -->.
it.next();
}
// Convert "-" + number to -number if allowed.
// Select 't' <--.
it.previous();
if (it.hasPrevious()) {
// Select 'prev1' <--.
Token prev1 = it.previous();
if (it.hasPrevious()) {
// Select 'prev2' <--.
Token prev2 = it.previous();
if (// Convert "± UNKNOWN".
t.type == TType.UNKNOWN && prev1.type.isPlusMinus() && // Don't convert "number/string/var ± ...".
!prev2.type.isIdentifier() && // Don't convert "func() ± ...".
prev2.type != TType.FUNC_END && // Don't convert "± @var".
!IVAR_PATTERN.matcher(t.val()).matches() && !VAR_PATTERN.matcher(t.val()).matches()) {
// Don't convert "± $var".
// It is a negative/positive number: Absorb the sign.
t.value = prev1.value + t.value;
// Select 'prev2' -->.
it.next();
// Select 'prev1' -->.
it.next();
// Remove 'prev1'.
it.remove();
} else {
// Select 'prev2' -->.
it.next();
// Select 'prev1' -->.
it.next();
}
} else {
// Select 'prev1' -->.
it.next();
}
}
// Select 't' -->.
it.next();
// Assign a type to all UNKNOWN tokens.
if (t.type == TType.UNKNOWN) {
if (t.val().charAt(0) == '/' && t.val().length() > 1) {
t.type = TType.COMMAND;
} else if (t.val().equals("$")) {
t.type = TType.FINAL_VAR;
} else if (VAR_PATTERN.matcher(t.val()).matches()) {
t.type = TType.VARIABLE;
} else if (IVAR_PATTERN.matcher(t.val()).matches()) {
t.type = TType.IVARIABLE;
} else if (t.val().charAt(0) == '@') {
throw new ConfigCompileException("IVariables must match the regex: " + IVAR_PATTERN, target);
} else if (keywords.contains(t.val())) {
t.type = TType.KEYWORD;
} else if (t.val().matches("[\t ]*")) {
t.type = TType.WHITESPACE;
} else {
t.type = TType.LIT;
}
}
// Skip this check if we're not in pure mscript.
if (inPureMScript) {
if (it.hasNext()) {
// Select 'next' -->.
Token next = it.next();
// Select 'next' <--.
it.previous();
// Select 't' <--.
it.previous();
if (t.type.isSymbol() && !t.type.isUnary() && !next.type.isUnary()) {
if (it.hasPrevious()) {
// Select 'prev1' <--.
Token prev1 = it.previous();
if (prev1.type.equals(TType.FUNC_START) || prev1.type.equals(TType.COMMA) || next.type.equals(TType.FUNC_END) || next.type.equals(TType.COMMA) || prev1.type.isSymbol() || next.type.isSymbol()) {
throw new ConfigCompileException("Unexpected symbol (" + t.val() + ")", t.getTarget());
}
// Select 'prev1' -->.
it.next();
}
}
// Select 't' -->.
it.next();
}
}
}
// Set file options
token_list.setFileOptions(fileOptions.toString());
// Make sure that the file options are the first non-comment code in the file
{
boolean foundCode = false;
for (Token t : token_list) {
if (t.type.isFileOption()) {
if (foundCode) {
throw new ConfigCompileException("File options must be the first non-comment section in the" + " code", t.target);
}
break;
}
if (!t.type.isComment() && !t.type.isWhitespace()) {
foundCode = true;
}
}
}
return token_list;
}
Also used :
TokenStream(com.laytonsmith.core.compiler.TokenStream)
Token(com.laytonsmith.core.constructs.Token)
CString(com.laytonsmith.core.constructs.CString)
ConfigCompileException(com.laytonsmith.core.exceptions.ConfigCompileException)
Target(com.laytonsmith.core.constructs.Target)
ListIterator(java.util.ListIterator)
Iterator(java.util.Iterator)
NoSuchElementException(java.util.NoSuchElementException)
Example 4 with Token
use of com.laytonsmith.core.constructs.Token in project CommandHelper by EngineHub.
the class CompilerObject method compile0.
void compile0() throws ConfigCompileException {
Token t = consume();
if (t.type == TType.NEWLINE) {
return;
}
if (t.type == TType.CONST_START) {
StringBuilder constName = new StringBuilder();
while ((t = consume()).type != TType.RCURLY_BRACKET) {
if (t.type != TType.BARE_STRING && t.type != TType.CONCAT) {
throw new ConfigCompileException("Constant names may only contain names and dots.", t.getTarget());
}
constName.append(t.val());
}
Construct constant = env.getConstant(constName.toString());
if (constant == null) {
throw new ConfigCompileException("Expected the constant ${" + constName.toString() + "} to be provided in the compilation options, but it wasn't.", t.getTarget());
}
t = new Token(TType.STRING, constant.val(), constant.getTarget());
}
if (t.type == TType.BARE_STRING && peek().type == TType.FUNC_START) {
consume();
CFunction f = new CFunction(t.val(), t.getTarget());
functionLines.add(peek().getTarget());
pushNode(f);
return;
}
if (t.type == TType.FUNC_END || t.type == TType.COMMA) {
if (autoConcatCounter > 0) {
autoConcatCounter--;
popNode(t.getTarget());
}
}
if (t.type == TType.COMMA) {
return;
}
if (t.type == TType.FUNC_END) {
// We're done with this child, so push it up
popNode(t.getTarget());
functionLines.pop();
return;
}
if (t.type == TType.LSQUARE_BRACKET) {
CFunction f = new CFunction("__cbracket__", Target.UNKNOWN);
pushNode(f);
bracketCounter++;
bracketLines.push(t.getTarget());
return;
}
if (t.type == TType.RSQUARE_BRACKET) {
if (bracketCounter == 0) {
throw new ConfigCompileException("Unexpected right bracket. (Did you have too many right square brackets (]) in your code?)", t.getTarget());
}
bracketCounter--;
bracketLines.pop();
popNode(t.getTarget());
return;
}
if (t.type == TType.LCURLY_BRACKET) {
CFunction f = new CFunction("__cbrace__", Target.UNKNOWN);
pushNode(f);
braceCounter++;
braceLines.push(t.getTarget());
return;
}
if (t.type == TType.RCURLY_BRACKET) {
if (braceCounter == 0) {
throw new ConfigCompileException("Unexpected right brace. (Did you have too many right braces (}) in your code?)", t.getTarget());
}
braceCounter--;
braceLines.pop();
popNode(t.getTarget());
return;
}
// If the next token ISN'T a ) , } ] we need to autoconcat this
if (peek().type != TType.FUNC_END && peek().type != TType.COMMA && peek().type != TType.RCURLY_BRACKET && peek().type != TType.RSQUARE_BRACKET) {
// ... unless we're already in an autoconcat
if (!(pointer.getData() instanceof CFunction && ((CFunction) pointer.getData()).val().equals("__autoconcat__"))) {
CFunction f = new CFunction("__autoconcat__", Target.UNKNOWN);
pushNode(f);
autoConcatCounter++;
}
}
if (t.type == TType.BARE_STRING && peek().type == TType.LABEL) {
consume();
pointer.addChild(new ParseTree(new CLabel(new CString(t.val(), t.getTarget())), stream.getFileOptions()));
return;
}
if (t.type.isIdentifier()) {
// If it's an atomic, put it in a construct and parse tree, then add it
pointer.addChild(new ParseTree(resolveIdentifier(t), stream.getFileOptions()));
return;
}
if (t.type.isSymbol()) {
pointer.addChild(new ParseTree(new CSymbol(t.val(), t.type, t.getTarget()), stream.getFileOptions()));
return;
}
// Now we have to check ahead for commas and other division parameters.
}
Also used :
CLabel(com.laytonsmith.core.constructs.CLabel)
CSymbol(com.laytonsmith.core.constructs.CSymbol)
Construct(com.laytonsmith.core.constructs.Construct)
CFunction(com.laytonsmith.core.constructs.CFunction)
Token(com.laytonsmith.core.constructs.Token)
ConfigCompileException(com.laytonsmith.core.exceptions.ConfigCompileException)
ParseTree(com.laytonsmith.core.ParseTree)
CString(com.laytonsmith.core.constructs.CString)
Example 5 with Token
use of com.laytonsmith.core.constructs.Token in project CommandHelper by EngineHub.
the class LexerObject method lex.
public TokenStream lex() throws ConfigCompileException {
if (token_list != null) {
return new TokenStream(new ArrayList<Token>(token_list), "");
} else {
token_list = new ArrayList<Token>();
}
for (int i = 0; i < config.length(); i++) {
Character c = config.charAt(i);
Character c2 = null;
Character c3 = null;
if (i < config.length() - 1) {
c2 = config.charAt(i + 1);
}
if (i < config.length() - 2) {
c3 = config.charAt(i + 2);
}
column += i - lastColumn;
lastColumn = i;
if (c == '\n') {
line_num++;
column = 1;
}
target = new Target(line_num, file, column);
// File Options
if (state_in_fileopts) {
if (c == '\\' && c2 == '>') {
// literal >
fileopts.append('>');
i++;
continue;
} else if (c == '>') {
state_in_fileopts = false;
continue;
} else {
fileopts.append(c);
continue;
}
}
// Comments are only applicable if we are not inside a string
if (!state_in_double_quote && !state_in_single_quote) {
// If we aren't already in a comment, we might be starting one here
if (!state_in_block_comment && !state_in_line_comment) {
if (c == '/' && c2 == '*') {
// Start of block comment
parseBuffer();
state_in_block_comment = true;
start_block_comment = line_num;
if (c3 == '*') {
// It's also a smart block comment
state_in_smart_block_comment = true;
i++;
} else {
state_in_smart_block_comment = false;
}
i++;
continue;
}
if (c == '#') {
parseBuffer();
// Start of line comment
state_in_line_comment = true;
continue;
}
} else if (state_in_block_comment) {
// We might be ending the block comment
if (c == '*' && c2 == '/') {
state_in_block_comment = false;
i++;
if (state_in_smart_block_comment) {
// We need to process the block comment here
// TODO:
// We need to process the block comment here
// TODO:
}
clearBuffer();
continue;
}
} else if (state_in_line_comment) {
if (c == '\n') {
state_in_line_comment = false;
clearBuffer();
continue;
}
}
}
// Ok, now if we are in a comment, we should just continue
if (state_in_block_comment || state_in_line_comment) {
if (state_in_smart_block_comment) {
buffer(c);
}
}
// Now we need to check for strings
if (!state_in_double_quote) {
if (c == '"') {
parseBuffer();
// Start of smart string
state_in_double_quote = true;
start_double_quote = line_num;
continue;
}
}
if (!state_in_single_quote) {
if (c == '\'') {
// Start of string
parseBuffer();
state_in_single_quote = true;
start_single_quote = line_num;
continue;
}
}
if (state_in_double_quote || state_in_single_quote) {
if (c == '\\') {
// It's an escaped something or another
switch(c2) {
case 'n':
buffer("\n");
i++;
break;
case 't':
buffer("\t");
i++;
break;
case 'u':
StringBuilder unicode = new StringBuilder();
for (int m = 0; m < 4; m++) {
try {
unicode.append(config.charAt(i + 2 + m));
} catch (IndexOutOfBoundsException e) {
// If this fails, they didn't put enough characters in the stream
error("Incomplete unicode escape");
}
}
try {
Integer.parseInt(unicode.toString(), 16);
} catch (NumberFormatException e) {
error("Unrecognized unicode escape sequence");
}
buffer(Character.toChars(Integer.parseInt(unicode.toString(), 16)));
i += 4;
break;
case '\'':
if (state_in_double_quote) {
// It's an error if we're in double quotes to escape a single quote
error("Invalid escape found. It is an error to escape single quotes inside a double quote.");
} else {
buffer("'");
i++;
}
break;
case '"':
if (state_in_single_quote) {
// It's an error if we're in single quotes to escape a double quote
error("Invalid escape found. It is an error to escape double quotes inside a single quote.");
} else {
buffer('"');
i++;
}
break;
default:
// It's invalid, so throw an exception
error("The escape sequence \\" + c2 + " is not a recognized escape sequence");
break;
}
continue;
}
}
// Now deal with ending a quote
if (state_in_double_quote) {
if (c == '"') {
state_in_double_quote = false;
append(clearBuffer(), Token.TType.SMART_STRING);
// This is currently an error, but won't be forever
error("Double quotes are currently unsupported");
continue;
} else {
buffer(c);
continue;
}
}
if (state_in_single_quote) {
if (c == '\'') {
state_in_single_quote = false;
append(clearBuffer(), Token.TType.STRING);
continue;
} else {
buffer(c);
continue;
}
}
// Now deal with multiline states
if (c == '>' && c2 == '>' && c3 == '>') {
// Multiline start
if (state_in_multiline) {
error("Found multiline start symbol while already in multiline!");
}
state_in_multiline = true;
start_multiline = line_num;
i += 2;
continue;
}
if (c == '<' && c2 == '<' && c3 == '<') {
if (!state_in_multiline) {
error("Found multiline end symbol while not in multiline!");
}
state_in_multiline = false;
i += 2;
continue;
}
// Newlines don't count
if (Character.isWhitespace(c) && c != '\n') {
// We need to parse the buffer
parseBuffer();
continue;
}
if (c == '<' && c2 == '!') {
if (!token_list.isEmpty()) {
throw new ConfigCompileException("File options must come first in the file.", target);
}
state_in_fileopts = true;
i++;
continue;
}
// get special handling up here, as well as square brackets
if (!state_in_pure_mscript) {
if (c == '[') {
if (state_in_opt_var) {
error("Found [ symbol, but a previous optional variable had already been started");
}
state_in_opt_var = true;
parseBuffer();
append("[", Token.TType.LSQUARE_BRACKET);
continue;
}
if (c == ']') {
if (!state_in_opt_var) {
error("Found ] symbol, but no optional variable had been started");
}
state_in_opt_var = false;
parseBuffer();
append("]", Token.TType.RSQUARE_BRACKET);
continue;
}
if (state_in_opt_var) {
if (c == '=') {
// This is an optional variable declaration
parseBuffer();
append("=", Token.TType.OPT_VAR_ASSIGN);
continue;
}
}
if (c == '=') {
state_in_pure_mscript = true;
parseBuffer();
append("=", Token.TType.ALIAS_END);
continue;
}
if (c == ':') {
parseBuffer();
append(":", Token.TType.LABEL);
continue;
}
if (c == '\n') {
parseBuffer();
if (token_list.isEmpty() || token_list.get(token_list.size() - 1).type != TType.NEWLINE) {
append("\n", TType.NEWLINE);
}
continue;
}
// At this point, all other tokens are to be taken literally
buffer(c);
continue;
}
// kept (except duplicate ones)
if (c == '\n') {
if (state_in_multiline) {
continue;
} else {
if (!token_list.isEmpty() && token_list.get(token_list.size() - 1).type != Token.TType.NEWLINE) {
parseBuffer();
if (usingNonPure) {
if (token_list.get(token_list.size() - 1).type != TType.NEWLINE) {
// Don't add duplicates
append("\n", Token.TType.NEWLINE);
}
// This also signals the end of pure mscript
state_in_pure_mscript = false;
continue;
} else if (state_in_pure_mscript) {
continue;
}
} else {
continue;
}
}
}
// Handle decimal place vs concat
if (c == '.' && Character.isDigit(c2)) {
// It'll get identified automatically in a bit
buffer(c);
continue;
}
// We need to handle /cmd vs division
if (c == '/' && (c2 == '/' || Character.isLetter(c2))) {
// It'll be registered as a bare string later
buffer(c);
continue;
}
// Now we are in pure mscript mode
// Loop through our token
int skip;
if ((skip = identifySymbol(i)) != -1) {
// Cool, it found one. Jump ahead.
i += skip;
continue;
}
buffer(c);
}
parseBuffer();
return new TokenStream(new ArrayList<Token>(token_list), fileopts.toString());
}
Also used :
Target(com.laytonsmith.core.constructs.Target)
Token(com.laytonsmith.core.constructs.Token)
ConfigCompileException(com.laytonsmith.core.exceptions.ConfigCompileException)
Aggregations
Token (com.laytonsmith.core.constructs.Token)14
ConfigCompileException (com.laytonsmith.core.exceptions.ConfigCompileException)8
CString (com.laytonsmith.core.constructs.CString)6
ArrayList (java.util.ArrayList)6
Variable (com.laytonsmith.core.constructs.Variable)4
TokenStream (com.laytonsmith.core.compiler.TokenStream)3
Construct (com.laytonsmith.core.constructs.Construct)3
IVariable (com.laytonsmith.core.constructs.IVariable)3
Target (com.laytonsmith.core.constructs.Target)3
ParseTree (com.laytonsmith.core.ParseTree)2
CFunction (com.laytonsmith.core.constructs.CFunction)2
CLabel (com.laytonsmith.core.constructs.CLabel)2
CSymbol (com.laytonsmith.core.constructs.CSymbol)2
ProfilePoint (com.laytonsmith.core.profiler.ProfilePoint)2
List (java.util.List)2
ListIterator (java.util.ListIterator)2
NoSuchElementException (java.util.NoSuchElementException)2
AtomicInteger (java.util.concurrent.atomic.AtomicInteger)2
FileOptions (com.laytonsmith.core.compiler.FileOptions)1
KeywordList (com.laytonsmith.core.compiler.KeywordList)1
