use of org.antlr.v4.runtime.dfa.DFA in project antlr4 by tunnelvisionlabs.
the class Parser method dumpDFA.
/**
* For debugging and other purposes.
*/
public void dumpDFA() {
boolean seenOne = false;
for (int d = 0; d < _interp.atn.decisionToDFA.length; d++) {
DFA dfa = _interp.atn.decisionToDFA[d];
if (!dfa.isEmpty()) {
if (seenOne)
System.out.println();
System.out.println("Decision " + dfa.decision + ":");
System.out.print(dfa.toString(getVocabulary(), getRuleNames()));
seenOne = true;
}
}
}
use of org.antlr.v4.runtime.dfa.DFA in project antlr4 by tunnelvisionlabs.
the class Parser method getDFAStrings.
/**
* For debugging and other purposes.
*/
public List<String> getDFAStrings() {
List<String> s = new ArrayList<String>();
for (int d = 0; d < _interp.atn.decisionToDFA.length; d++) {
DFA dfa = _interp.atn.decisionToDFA[d];
s.add(dfa.toString(getVocabulary(), getRuleNames()));
}
return s;
}
use of org.antlr.v4.runtime.dfa.DFA in project antlr4 by tunnelvisionlabs.
the class LexerATNSimulator method execATN.
protected int execATN(@NotNull CharStream input, @NotNull DFAState ds0) {
// System.out.println("enter exec index "+input.index()+" from "+ds0.configs);
if (debug) {
System.out.format(Locale.getDefault(), "start state closure=%s\n", ds0.configs);
}
if (ds0.isAcceptState()) {
// allow zero-length tokens
captureSimState(prevAccept, input, ds0);
}
int t = input.LA(1);
@NotNull DFAState // s is current/from DFA state
s = ds0;
while (true) {
// while more work
if (debug) {
System.out.format(Locale.getDefault(), "execATN loop starting closure: %s\n", s.configs);
}
// As we move src->trg, src->trg, we keep track of the previous trg to
// avoid looking up the DFA state again, which is expensive.
// If the previous target was already part of the DFA, we might
// be able to avoid doing a reach operation upon t. If s!=null,
// it means that semantic predicates didn't prevent us from
// creating a DFA state. Once we know s!=null, we check to see if
// the DFA state has an edge already for t. If so, we can just reuse
// it's configuration set; there's no point in re-computing it.
// This is kind of like doing DFA simulation within the ATN
// simulation because DFA simulation is really just a way to avoid
// computing reach/closure sets. Technically, once we know that
// we have a previously added DFA state, we could jump over to
// the DFA simulator. But, that would mean popping back and forth
// a lot and making things more complicated algorithmically.
// This optimization makes a lot of sense for loops within DFA.
// A character will take us back to an existing DFA state
// that already has lots of edges out of it. e.g., .* in comments.
DFAState target = getExistingTargetState(s, t);
if (target == null) {
target = computeTargetState(input, s, t);
}
if (target == ERROR) {
break;
}
// end of the token.
if (t != IntStream.EOF) {
consume(input);
}
if (target.isAcceptState()) {
captureSimState(prevAccept, input, target);
if (t == IntStream.EOF) {
break;
}
}
t = input.LA(1);
// flip; current DFA target becomes new src/from state
s = target;
}
return failOrAccept(prevAccept, input, s.configs, t);
}
use of org.antlr.v4.runtime.dfa.DFA in project antlr4 by tunnelvisionlabs.
the class ATN method defineDecisionState.
public int defineDecisionState(@NotNull DecisionState s) {
decisionToState.add(s);
s.decision = decisionToState.size() - 1;
decisionToDFA = Arrays.copyOf(decisionToDFA, decisionToState.size());
decisionToDFA[decisionToDFA.length - 1] = new DFA(s, s.decision);
return s.decision;
}
use of org.antlr.v4.runtime.dfa.DFA in project antlr4 by tunnelvisionlabs.
the class TestPerformance method parseSources.
@SuppressWarnings("unused")
protected void parseSources(final int currentPass, final ParserFactory factory, Collection<InputDescriptor> sources, boolean shuffleSources) throws InterruptedException {
if (shuffleSources) {
List<InputDescriptor> sourcesList = new ArrayList<InputDescriptor>(sources);
synchronized (RANDOM) {
Collections.shuffle(sourcesList, RANDOM);
}
sources = sourcesList;
}
long startTime = System.nanoTime();
tokenCount.set(currentPass, 0);
int inputSize = 0;
int inputCount = 0;
Collection<Future<FileParseResult>> results = new ArrayList<Future<FileParseResult>>();
ExecutorService executorService;
if (FILE_GRANULARITY) {
executorService = Executors.newFixedThreadPool(FILE_GRANULARITY ? NUMBER_OF_THREADS : 1, new NumberedThreadFactory());
} else {
executorService = Executors.newSingleThreadExecutor(new FixedThreadNumberFactory(((NumberedThread) Thread.currentThread()).getThreadNumber()));
}
for (InputDescriptor inputDescriptor : sources) {
if (inputCount >= MAX_FILES_PER_PARSE_ITERATION) {
break;
}
final CharStream input = inputDescriptor.getInputStream();
input.seek(0);
inputSize += input.size();
inputCount++;
Future<FileParseResult> futureChecksum = executorService.submit(new Callable<FileParseResult>() {
@Override
public FileParseResult call() {
// System.out.format("Parsing file %s\n", input.getSourceName());
try {
return factory.parseFile(input, currentPass, ((NumberedThread) Thread.currentThread()).getThreadNumber());
} catch (IllegalStateException ex) {
ex.printStackTrace(System.err);
} catch (Throwable t) {
t.printStackTrace(System.err);
}
return null;
}
});
results.add(futureChecksum);
}
MurmurHashChecksum checksum = new MurmurHashChecksum();
int currentIndex = -1;
for (Future<FileParseResult> future : results) {
currentIndex++;
int fileChecksum = 0;
try {
FileParseResult fileResult = future.get();
if (COMPUTE_TRANSITION_STATS) {
totalTransitionsPerFile[currentPass][currentIndex] = sum(fileResult.parserTotalTransitions);
computedTransitionsPerFile[currentPass][currentIndex] = sum(fileResult.parserComputedTransitions);
if (DETAILED_DFA_STATE_STATS) {
decisionInvocationsPerFile[currentPass][currentIndex] = fileResult.decisionInvocations;
fullContextFallbackPerFile[currentPass][currentIndex] = fileResult.fullContextFallback;
nonSllPerFile[currentPass][currentIndex] = fileResult.nonSll;
totalTransitionsPerDecisionPerFile[currentPass][currentIndex] = fileResult.parserTotalTransitions;
computedTransitionsPerDecisionPerFile[currentPass][currentIndex] = fileResult.parserComputedTransitions;
fullContextTransitionsPerDecisionPerFile[currentPass][currentIndex] = fileResult.parserFullContextTransitions;
}
}
if (COMPUTE_TIMING_STATS) {
timePerFile[currentPass][currentIndex] = fileResult.endTime - fileResult.startTime;
tokensPerFile[currentPass][currentIndex] = fileResult.tokenCount;
}
fileChecksum = fileResult.checksum;
} catch (ExecutionException ex) {
Logger.getLogger(TestPerformance.class.getName()).log(Level.SEVERE, null, ex);
}
if (COMPUTE_CHECKSUM) {
updateChecksum(checksum, fileChecksum);
}
}
executorService.shutdown();
executorService.awaitTermination(Long.MAX_VALUE, TimeUnit.NANOSECONDS);
System.out.format("%d. Total parse time for %d files (%d KB, %d tokens%s): %.0fms%n", currentPass + 1, inputCount, inputSize / 1024, tokenCount.get(currentPass), COMPUTE_CHECKSUM ? String.format(", checksum 0x%8X", checksum.getValue()) : "", (double) (System.nanoTime() - startTime) / 1000000.0);
if (sharedLexers.length > 0) {
int index = FILE_GRANULARITY ? 0 : ((NumberedThread) Thread.currentThread()).getThreadNumber();
Lexer lexer = sharedLexers[index];
final LexerATNSimulator lexerInterpreter = lexer.getInterpreter();
final DFA[] modeToDFA = lexerInterpreter.atn.modeToDFA;
if (SHOW_DFA_STATE_STATS) {
int states = 0;
int configs = 0;
Set<ATNConfig> uniqueConfigs = new HashSet<ATNConfig>();
for (int i = 0; i < modeToDFA.length; i++) {
DFA dfa = modeToDFA[i];
if (dfa == null) {
continue;
}
states += dfa.states.size();
for (DFAState state : dfa.states.values()) {
configs += state.configs.size();
uniqueConfigs.addAll(state.configs);
}
}
System.out.format("There are %d lexer DFAState instances, %d configs (%d unique), %d prediction contexts.%n", states, configs, uniqueConfigs.size(), lexerInterpreter.atn.getContextCacheSize());
if (DETAILED_DFA_STATE_STATS) {
System.out.format("\tMode\tStates\tConfigs\tMode%n");
for (int i = 0; i < modeToDFA.length; i++) {
DFA dfa = modeToDFA[i];
if (dfa == null || dfa.states.isEmpty()) {
continue;
}
int modeConfigs = 0;
for (DFAState state : dfa.states.values()) {
modeConfigs += state.configs.size();
}
String modeName = lexer.getModeNames()[i];
System.out.format("\t%d\t%d\t%d\t%s%n", dfa.decision, dfa.states.size(), modeConfigs, modeName);
}
}
}
}
if (RUN_PARSER && sharedParsers.length > 0) {
int index = FILE_GRANULARITY ? 0 : ((NumberedThread) Thread.currentThread()).getThreadNumber();
Parser parser = sharedParsers[index];
// make sure the individual DFAState objects actually have unique ATNConfig arrays
final ParserATNSimulator interpreter = parser.getInterpreter();
final DFA[] decisionToDFA = interpreter.atn.decisionToDFA;
if (SHOW_DFA_STATE_STATS) {
int states = 0;
int configs = 0;
Set<ATNConfig> uniqueConfigs = new HashSet<ATNConfig>();
for (int i = 0; i < decisionToDFA.length; i++) {
DFA dfa = decisionToDFA[i];
if (dfa == null) {
continue;
}
states += dfa.states.size();
for (DFAState state : dfa.states.values()) {
configs += state.configs.size();
uniqueConfigs.addAll(state.configs);
}
}
System.out.format("There are %d parser DFAState instances, %d configs (%d unique), %d prediction contexts.%n", states, configs, uniqueConfigs.size(), interpreter.atn.getContextCacheSize());
if (DETAILED_DFA_STATE_STATS) {
if (COMPUTE_TRANSITION_STATS) {
System.out.format("\tDecision\tStates\tConfigs\tPredict (ALL)\tPredict (LL)\tNon-SLL\tTransitions\tTransitions (ATN)\tTransitions (LL)\tLA (SLL)\tLA (LL)\tRule%n");
} else {
System.out.format("\tDecision\tStates\tConfigs\tRule%n");
}
for (int i = 0; i < decisionToDFA.length; i++) {
DFA dfa = decisionToDFA[i];
if (dfa == null || dfa.states.isEmpty()) {
continue;
}
int decisionConfigs = 0;
for (DFAState state : dfa.states.values()) {
decisionConfigs += state.configs.size();
}
String ruleName = parser.getRuleNames()[parser.getATN().decisionToState.get(dfa.decision).ruleIndex];
long calls = 0;
long fullContextCalls = 0;
long nonSllCalls = 0;
long transitions = 0;
long computedTransitions = 0;
long fullContextTransitions = 0;
double lookahead = 0;
double fullContextLookahead = 0;
String formatString;
if (COMPUTE_TRANSITION_STATS) {
for (long[] data : decisionInvocationsPerFile[currentPass]) {
calls += data[i];
}
for (long[] data : fullContextFallbackPerFile[currentPass]) {
fullContextCalls += data[i];
}
for (long[] data : nonSllPerFile[currentPass]) {
nonSllCalls += data[i];
}
for (long[] data : totalTransitionsPerDecisionPerFile[currentPass]) {
transitions += data[i];
}
for (long[] data : computedTransitionsPerDecisionPerFile[currentPass]) {
computedTransitions += data[i];
}
for (long[] data : fullContextTransitionsPerDecisionPerFile[currentPass]) {
fullContextTransitions += data[i];
}
if (calls > 0) {
lookahead = (double) (transitions - fullContextTransitions) / (double) calls;
}
if (fullContextCalls > 0) {
fullContextLookahead = (double) fullContextTransitions / (double) fullContextCalls;
}
formatString = "\t%1$d\t%2$d\t%3$d\t%4$d\t%5$d\t%6$d\t%7$d\t%8$d\t%9$d\t%10$f\t%11$f\t%12$s%n";
} else {
calls = 0;
formatString = "\t%1$d\t%2$d\t%3$d\t%12$s%n";
}
System.out.format(formatString, dfa.decision, dfa.states.size(), decisionConfigs, calls, fullContextCalls, nonSllCalls, transitions, computedTransitions, fullContextTransitions, lookahead, fullContextLookahead, ruleName);
}
}
}
int localDfaCount = 0;
int globalDfaCount = 0;
int localConfigCount = 0;
int globalConfigCount = 0;
int[] contextsInDFAState = new int[0];
for (int i = 0; i < decisionToDFA.length; i++) {
DFA dfa = decisionToDFA[i];
if (dfa == null) {
continue;
}
if (SHOW_CONFIG_STATS) {
for (DFAState state : dfa.states.keySet()) {
if (state.configs.size() >= contextsInDFAState.length) {
contextsInDFAState = Arrays.copyOf(contextsInDFAState, state.configs.size() + 1);
}
if (state.isAcceptState()) {
boolean hasGlobal = false;
for (ATNConfig config : state.configs) {
if (config.getReachesIntoOuterContext()) {
globalConfigCount++;
hasGlobal = true;
} else {
localConfigCount++;
}
}
if (hasGlobal) {
globalDfaCount++;
} else {
localDfaCount++;
}
}
contextsInDFAState[state.configs.size()]++;
}
}
if (EXPORT_LARGEST_CONFIG_CONTEXTS) {
for (DFAState state : dfa.states.keySet()) {
for (ATNConfig config : state.configs) {
String configOutput = config.toDotString();
if (configOutput.length() <= configOutputSize) {
continue;
}
configOutputSize = configOutput.length();
writeFile(tmpdir, "d" + dfa.decision + ".s" + state.stateNumber + ".a" + config.getAlt() + ".config.dot", configOutput);
}
}
}
}
if (SHOW_CONFIG_STATS && currentPass == 0) {
System.out.format(" DFA accept states: %d total, %d with only local context, %d with a global context%n", localDfaCount + globalDfaCount, localDfaCount, globalDfaCount);
System.out.format(" Config stats: %d total, %d local, %d global%n", localConfigCount + globalConfigCount, localConfigCount, globalConfigCount);
if (SHOW_DFA_STATE_STATS) {
for (int i = 0; i < contextsInDFAState.length; i++) {
if (contextsInDFAState[i] != 0) {
System.out.format(" %d configs = %d%n", i, contextsInDFAState[i]);
}
}
}
}
}
if (COMPUTE_TIMING_STATS) {
System.out.format("File\tTokens\tTime%n");
for (int i = 0; i < timePerFile[currentPass].length; i++) {
System.out.format("%d\t%d\t%d%n", i + 1, tokensPerFile[currentPass][i], timePerFile[currentPass][i]);
}
}
}
Aggregations