1465 lines
55 KiB
Java
1465 lines
55 KiB
Java
/*
|
|
* Copyright © 2002 Sun Microsystems, Inc., 4150 Network Circle, Santa Clara,
|
|
* California 95054, U.S.A. All rights reserved. Sun Microsystems, Inc. has
|
|
* intellectual property rights relating to technology embodied in the product
|
|
* that is described in this document. In particular, and without limitation,
|
|
* these intellectual property rights may include one or more of the U.S.
|
|
* patents listed at http://www.sun.com/patents and one or more additional
|
|
* patents or pending patent applications in the U.S. and in other countries.
|
|
* U.S. Government Rights - Commercial software. Government users are subject
|
|
* to the Sun Microsystems, Inc. standard license agreement and applicable
|
|
* provisions of the FAR and its supplements. Use is subject to license terms.
|
|
* Sun, Sun Microsystems, the Sun logo and Java are trademarks or registered
|
|
* trademarks of Sun Microsystems, Inc. in the U.S. and other countries. This
|
|
* product is covered and controlled by U.S. Export Control laws and may be
|
|
* subject to the export or import laws in other countries. Nuclear, missile,
|
|
* chemical biological weapons or nuclear maritime end uses or end users,
|
|
* whether direct or indirect, are strictly prohibited. Export or reexport
|
|
* to countries subject to U.S. embargo or to entities identified on U.S.
|
|
* export exclusion lists, including, but not limited to, the denied persons
|
|
* and specially designated nationals lists is strictly prohibited.
|
|
*/
|
|
|
|
package csjavacc.parser;
|
|
|
|
import java.util.*;
|
|
import java.io.*;
|
|
|
|
import csjavacc.struct.Action;
|
|
import csjavacc.struct.Nfa;
|
|
import csjavacc.struct.RegularExpression;
|
|
import csjavacc.struct.TokenProduction;
|
|
|
|
|
|
public class LexGen extends CSJavaCCGlobals implements CSJavaCCParserConstants{
|
|
static private java.io.PrintWriter ostr;
|
|
static private String staticString;
|
|
static private String tokMgrClassName;
|
|
|
|
// Hashtable of vectors
|
|
static Hashtable allTpsForState = new Hashtable();
|
|
public static int lexStateIndex = 0;
|
|
static int[] kinds;
|
|
public static int maxOrdinal = 1;
|
|
public static String lexStateSuffix;
|
|
static String[] newLexState;
|
|
public static int[] lexStates;
|
|
public static boolean[] ignoreCase;
|
|
public static Action[] actions;
|
|
public static Hashtable initStates = new Hashtable();
|
|
public static int stateSetSize;
|
|
public static int maxLexStates;
|
|
public static String[] lexStateName;
|
|
static NfaState[] singlesToSkip;
|
|
public static long[] toSkip;
|
|
public static long[] toSpecial;
|
|
public static long[] toMore;
|
|
public static long[] toToken;
|
|
public static int defaultLexState;
|
|
public static RegularExpression[] rexprs;
|
|
public static int[] maxLongsReqd;
|
|
public static int[] initMatch;
|
|
public static int[] canMatchAnyChar;
|
|
public static boolean hasEmptyMatch;
|
|
public static boolean[] canLoop;
|
|
public static boolean[] stateHasActions;
|
|
public static boolean hasLoop = false;
|
|
public static boolean[] canReachOnMore;
|
|
public static boolean[] hasNfa;
|
|
public static boolean[] mixed;
|
|
public static NfaState initialState;
|
|
public static int curKind;
|
|
static boolean hasSkipActions = false;
|
|
static boolean hasMoreActions = false;
|
|
static boolean hasTokenActions = false;
|
|
static boolean hasSpecial = false;
|
|
static boolean hasSkip = false;
|
|
static boolean hasMore = false;
|
|
static boolean hasToken = false;
|
|
public static RegularExpression curRE;
|
|
public static boolean keepLineCol;
|
|
static boolean namespace = false;
|
|
|
|
static void PrintClassHead()
|
|
{
|
|
int i, j;
|
|
try {
|
|
File tmp = new File(Options.getOutputDirectory(), tokMgrClassName + extension);
|
|
ostr = new java.io.PrintWriter(
|
|
new java.io.BufferedWriter(
|
|
new java.io.FileWriter(tmp),
|
|
8092
|
|
)
|
|
);
|
|
Vector tn = (Vector)(toolNames.clone());
|
|
tn.addElement(toolName);
|
|
|
|
ostr.println("/* " + getIdString(tn, tokMgrClassName + extension) + " */");
|
|
|
|
int l = 0, kind;
|
|
i = 1;
|
|
boolean namehit = false;
|
|
for (;;){
|
|
if (cu_to_insertion_point_1.size() <= l)
|
|
break;
|
|
|
|
kind = ((Token)cu_to_insertion_point_1.elementAt(l)).kind;
|
|
if(kind == NAMESPACE || kind == IMPORT) {
|
|
if(kind == NAMESPACE){
|
|
namespace = true;
|
|
namehit = true;
|
|
}
|
|
for (; i < cu_to_insertion_point_1.size(); i++) {
|
|
kind = ((Token)cu_to_insertion_point_1.elementAt(i)).kind;
|
|
if (kind == SEMICOLON || kind == ABSTRACT ||
|
|
kind == OVERRIDE || kind == PUBLIC ||
|
|
kind == CLASS || kind == INTERFACE)
|
|
{
|
|
cline = ((Token)(cu_to_insertion_point_1.elementAt(l))).beginLine;
|
|
ccol = ((Token)(cu_to_insertion_point_1.elementAt(l))).beginColumn;
|
|
for (j = l; j < i; j++) {
|
|
printToken((Token)(cu_to_insertion_point_1.elementAt(j)), ostr);
|
|
}
|
|
if (kind == SEMICOLON){
|
|
if(namehit){
|
|
ostr.println("{");
|
|
namehit = false;
|
|
}else
|
|
printToken((Token)(cu_to_insertion_point_1.elementAt(j)), ostr);
|
|
}
|
|
ostr.println("");
|
|
break;
|
|
}
|
|
}
|
|
l = ++i;
|
|
}
|
|
else
|
|
break;
|
|
}
|
|
|
|
ostr.println("");
|
|
ostr.println("public class " + tokMgrClassName + " : " +
|
|
cu_name + "Constants");
|
|
ostr.println("{"); // }
|
|
}
|
|
catch (java.io.IOException err) {
|
|
CSJavaCCErrors.semantic_error("Could not create file : " + tokMgrClassName + extension+"\n");
|
|
throw new Error();
|
|
}
|
|
|
|
if (token_mgr_decls != null && token_mgr_decls.size() > 0)
|
|
{
|
|
Token t = (Token)token_mgr_decls.elementAt(0);
|
|
boolean commonTokenActionSeen = false;
|
|
boolean commonTokenActionNeeded = Options.getCommonTokenAction();
|
|
|
|
printTokenSetup((Token)token_mgr_decls.elementAt(0));
|
|
ccol = 1;
|
|
|
|
for (j = 0; j < token_mgr_decls.size(); j++)
|
|
{
|
|
t = (Token)token_mgr_decls.elementAt(j);
|
|
if (t.kind == IDENTIFIER &&
|
|
commonTokenActionNeeded &&
|
|
!commonTokenActionSeen)
|
|
commonTokenActionSeen = t.image.equals("CommonTokenAction");
|
|
|
|
printToken(t, ostr);
|
|
}
|
|
|
|
ostr.println("");
|
|
if (commonTokenActionNeeded && !commonTokenActionSeen)
|
|
CSJavaCCErrors.warning("You have the COMMON_TOKEN_ACTION option set. But it appears you have not defined the method :\n"+
|
|
" " + staticString + "void CommonTokenAction(Token t)\n" +
|
|
"in your TOKEN_MGR_DECLS. The generated token manager will not compile.");
|
|
|
|
}
|
|
else if (Options.getCommonTokenAction())
|
|
{
|
|
CSJavaCCErrors.warning("You have the COMMON_TOKEN_ACTION option set. But you have not defined the method :\n"+
|
|
" " + staticString + "void CommonTokenAction(Token t)\n" +
|
|
"in your TOKEN_MGR_DECLS. The generated token manager will not compile.");
|
|
}
|
|
|
|
ostr.println(" public " + staticString + " System.IO.TextWriter debugStream = new System.IO.StreamWriter(System.Console.OpenStandardError());");
|
|
ostr.println(" public " + staticString + " void setDebugStream(System.IO.TextWriter ds) { debugStream = ds; }");
|
|
|
|
if(Options.getTokenManagerUsesParser() && !Options.getStatic()){
|
|
ostr.println(" public " + cu_name + " parser = null;");
|
|
}
|
|
}
|
|
|
|
static void DumpDebugMethods()
|
|
{
|
|
|
|
ostr.println(" " + staticString + " int kindCnt = 0;");
|
|
ostr.println(" protected " + staticString + " String jjKindsForBitVector(int i, long vec){");
|
|
ostr.println(" String retVal = \"\";");
|
|
ostr.println(" if (i == 0)");
|
|
ostr.println(" kindCnt = 0;");
|
|
ostr.println(" for (int j = 0; j < 64; j++){");
|
|
ostr.println(" if ((vec & (1L << j)) != 0L){");
|
|
ostr.println(" if (kindCnt++ > 0)");
|
|
ostr.println(" retVal += \", \";");
|
|
ostr.println(" if (kindCnt % 5 == 0)");
|
|
ostr.println(" retVal += \"\\n \";");
|
|
ostr.println(" retVal += tokenImage[i * 64 + j];");
|
|
ostr.println(" }");
|
|
ostr.println(" }");
|
|
ostr.println(" return retVal;");
|
|
ostr.println(" }");
|
|
ostr.println("");
|
|
|
|
ostr.println(" protected " + staticString + " String jjKindsForStateVector(int lexState, int[] vec, int start, int end){");
|
|
ostr.println(" bool[] kindDone = new bool[" + maxOrdinal + "];");
|
|
ostr.println(" String retVal = \"\";");
|
|
ostr.println(" int cnt = 0;");
|
|
ostr.println(" for (int i = start; i < end; i++){");
|
|
ostr.println(" if (vec[i] == -1)");
|
|
ostr.println(" continue;");
|
|
ostr.println(" int[] stateSet = statesForState[curLexState][vec[i]];");
|
|
ostr.println(" for (int j = 0; j < stateSet.Length; j++){");
|
|
ostr.println(" int state = stateSet[j];");
|
|
ostr.println(" if (!kindDone[kindForState[lexState][state]]){");
|
|
ostr.println(" kindDone[kindForState[lexState][state]] = true;");
|
|
ostr.println(" if (cnt++ > 0)");
|
|
ostr.println(" retVal += \", \";");
|
|
ostr.println(" if (cnt % 5 == 0)");
|
|
ostr.println(" retVal += \"\\n \";");
|
|
ostr.println(" retVal += tokenImage[kindForState[lexState][state]];");
|
|
ostr.println(" }");
|
|
ostr.println(" }");
|
|
ostr.println(" }");
|
|
ostr.println(" if (cnt == 0)");
|
|
ostr.println(" return \"{ }\";");
|
|
ostr.println(" else");
|
|
ostr.println(" return \"{ \" + retVal + \" }\";");
|
|
ostr.println(" }");
|
|
ostr.println("");
|
|
}
|
|
|
|
static void BuildLexStatesTable()
|
|
{
|
|
Enumeration e = rexprlist.elements();
|
|
TokenProduction tp;
|
|
int i;
|
|
|
|
String[] tmpLexStateName = new String[lexstate_I2S.size()];
|
|
while (e.hasMoreElements())
|
|
{
|
|
tp = (TokenProduction)e.nextElement();
|
|
Vector respecs = tp.respecs;
|
|
Vector tps;
|
|
|
|
for (i = 0; i < tp.lexStates.length; i++)
|
|
{
|
|
if ((tps = (Vector)allTpsForState.get(tp.lexStates[i])) == null)
|
|
{
|
|
tmpLexStateName[maxLexStates++] = tp.lexStates[i];
|
|
allTpsForState.put(tp.lexStates[i], tps = new Vector());
|
|
}
|
|
|
|
tps.addElement(tp);
|
|
}
|
|
|
|
if (respecs == null || respecs.size() == 0)
|
|
continue;
|
|
|
|
RegularExpression re;
|
|
for (i = 0; i < respecs.size(); i++)
|
|
if (maxOrdinal <= (re = ((RegExprSpec)respecs.elementAt(i)).rexp).ordinal)
|
|
maxOrdinal = re.ordinal + 1;
|
|
}
|
|
|
|
kinds = new int[maxOrdinal];
|
|
toSkip = new long[maxOrdinal / 64 + 1];
|
|
toSpecial = new long[maxOrdinal / 64 + 1];
|
|
toMore = new long[maxOrdinal / 64 + 1];
|
|
toToken = new long[maxOrdinal / 64 + 1];
|
|
toToken[0] = 1L;
|
|
actions = new Action[maxOrdinal];
|
|
actions[0] = actForEof;
|
|
hasTokenActions = actForEof != null;
|
|
initStates = new Hashtable();
|
|
canMatchAnyChar = new int[maxLexStates];
|
|
canLoop = new boolean[maxLexStates];
|
|
stateHasActions = new boolean[maxLexStates];
|
|
lexStateName = new String[maxLexStates];
|
|
singlesToSkip = new NfaState[maxLexStates];
|
|
System.arraycopy(tmpLexStateName, 0, lexStateName, 0, maxLexStates);
|
|
|
|
for (i = 0; i < maxLexStates; i++)
|
|
canMatchAnyChar[i] = -1;
|
|
|
|
hasNfa = new boolean[maxLexStates];
|
|
mixed = new boolean[maxLexStates];
|
|
maxLongsReqd = new int[maxLexStates];
|
|
initMatch = new int[maxLexStates];
|
|
newLexState = new String[maxOrdinal];
|
|
newLexState[0] = nextStateForEof;
|
|
hasEmptyMatch = false;
|
|
lexStates = new int[maxOrdinal];
|
|
ignoreCase = new boolean[maxOrdinal];
|
|
rexprs = new RegularExpression[maxOrdinal];
|
|
RStringLiteral.allImages = new String[maxOrdinal];
|
|
canReachOnMore = new boolean[maxLexStates];
|
|
}
|
|
|
|
static int GetIndex(String name)
|
|
{
|
|
for (int i = 0; i < lexStateName.length; i++)
|
|
if (lexStateName[i] != null && lexStateName[i].equals(name))
|
|
return i;
|
|
|
|
throw new Error(); // Should never come here
|
|
}
|
|
|
|
public static void AddCharToSkip(char c, int kind)
|
|
{
|
|
singlesToSkip[lexStateIndex].AddChar(c);
|
|
singlesToSkip[lexStateIndex].kind = kind;
|
|
}
|
|
|
|
public static void start()
|
|
{
|
|
if (!Options.getBuildTokenManager() ||
|
|
Options.getUserTokenManager() ||
|
|
CSJavaCCErrors.get_error_count() > 0)
|
|
return;
|
|
|
|
keepLineCol = Options.getKeepLineColumn();
|
|
Vector choices = new Vector();
|
|
Enumeration e;
|
|
TokenProduction tp;
|
|
int i, j;
|
|
|
|
staticString = (Options.getStatic() ? "static " : "");
|
|
tokMgrClassName = cu_name + "TokenManager";
|
|
|
|
PrintClassHead();
|
|
BuildLexStatesTable();
|
|
|
|
e = allTpsForState.keys();
|
|
|
|
boolean ignoring = false;
|
|
|
|
while (e.hasMoreElements())
|
|
{
|
|
NfaState.ReInit();
|
|
RStringLiteral.ReInit();
|
|
|
|
String key = (String)e.nextElement();
|
|
|
|
lexStateIndex = GetIndex(key);
|
|
lexStateSuffix = "_" + lexStateIndex;
|
|
Vector allTps = (Vector)allTpsForState.get(key);
|
|
initStates.put(key, initialState = new NfaState());
|
|
ignoring = false;
|
|
|
|
singlesToSkip[lexStateIndex] = new NfaState();
|
|
singlesToSkip[lexStateIndex].dummy = true;
|
|
|
|
if (key.equals("DEFAULT"))
|
|
defaultLexState = lexStateIndex;
|
|
|
|
for (i = 0; i < allTps.size(); i++)
|
|
{
|
|
tp = (TokenProduction)allTps.elementAt(i);
|
|
int kind = tp.kind;
|
|
boolean ignore = tp.ignoreCase;
|
|
Vector rexps = tp.respecs;
|
|
|
|
if (i == 0)
|
|
ignoring = ignore;
|
|
|
|
for (j = 0; j < rexps.size(); j++)
|
|
{
|
|
RegExprSpec respec = (RegExprSpec)rexps.elementAt(j);
|
|
curRE = respec.rexp;
|
|
|
|
rexprs[curKind = curRE.ordinal] = curRE;
|
|
lexStates[curRE.ordinal] = lexStateIndex;
|
|
ignoreCase[curRE.ordinal] = ignore;
|
|
|
|
if (curRE.private_rexp)
|
|
{
|
|
kinds[curRE.ordinal] = -1;
|
|
continue;
|
|
}
|
|
|
|
if (curRE instanceof RStringLiteral &&
|
|
!((RStringLiteral)curRE).image.equals(""))
|
|
{
|
|
((RStringLiteral)curRE).GenerateDfa(ostr, curRE.ordinal);
|
|
if (i != 0 && !mixed[lexStateIndex] && ignoring != ignore)
|
|
mixed[lexStateIndex] = true;
|
|
}
|
|
else if (curRE.CanMatchAnyChar())
|
|
{
|
|
if (canMatchAnyChar[lexStateIndex] == -1 ||
|
|
canMatchAnyChar[lexStateIndex] > curRE.ordinal)
|
|
canMatchAnyChar[lexStateIndex] = curRE.ordinal;
|
|
}
|
|
else
|
|
{
|
|
Nfa temp;
|
|
|
|
if (curRE instanceof RChoice)
|
|
choices.addElement(curRE);
|
|
|
|
temp = curRE.GenerateNfa(ignore);
|
|
temp.end.isFinal = true;
|
|
temp.end.kind = curRE.ordinal;
|
|
initialState.AddMove(temp.start);
|
|
}
|
|
|
|
if (kinds.length < curRE.ordinal)
|
|
{
|
|
int[] tmp = new int[curRE.ordinal + 1];
|
|
|
|
System.arraycopy(kinds, 0, tmp, 0, kinds.length);
|
|
kinds = tmp;
|
|
}
|
|
//System.err.println(" ordinal : " + curRE.ordinal);
|
|
|
|
kinds[curRE.ordinal] = kind;
|
|
|
|
if (respec.nextState != null &&
|
|
!respec.nextState.equals(lexStateName[lexStateIndex]))
|
|
newLexState[curRE.ordinal] = respec.nextState;
|
|
|
|
if (respec.act != null && respec.act.action_tokens != null &&
|
|
respec.act.action_tokens.size() > 0)
|
|
actions[curRE.ordinal] = respec.act;
|
|
|
|
switch(kind)
|
|
{
|
|
case TokenProduction.SPECIAL :
|
|
hasSkipActions |= (actions[curRE.ordinal] != null) ||
|
|
(newLexState[curRE.ordinal] != null);
|
|
hasSpecial = true;
|
|
toSpecial[curRE.ordinal / 64] |= 1L << (curRE.ordinal % 64);
|
|
toSkip[curRE.ordinal / 64] |= 1L << (curRE.ordinal % 64);
|
|
break;
|
|
case TokenProduction.SKIP :
|
|
hasSkipActions |= (actions[curRE.ordinal] != null);
|
|
hasSkip = true;
|
|
toSkip[curRE.ordinal / 64] |= 1L << (curRE.ordinal % 64);
|
|
break;
|
|
case TokenProduction.MORE :
|
|
hasMoreActions |= (actions[curRE.ordinal] != null);
|
|
hasMore = true;
|
|
toMore[curRE.ordinal / 64] |= 1L << (curRE.ordinal % 64);
|
|
|
|
if (newLexState[curRE.ordinal] != null)
|
|
canReachOnMore[GetIndex(newLexState[curRE.ordinal])] = true;
|
|
else
|
|
canReachOnMore[lexStateIndex] = true;
|
|
|
|
break;
|
|
case TokenProduction.TOKEN :
|
|
hasTokenActions |= (actions[curRE.ordinal] != null);
|
|
hasToken = true;
|
|
toToken[curRE.ordinal / 64] |= 1L << (curRE.ordinal % 64);
|
|
break;
|
|
}
|
|
}
|
|
}
|
|
|
|
// Generate a static block for initializing the nfa transitions
|
|
NfaState.ComputeClosures();
|
|
|
|
for (i = 0; i < initialState.epsilonMoves.size(); i++)
|
|
((NfaState)initialState.epsilonMoves.elementAt(i)).GenerateCode();
|
|
|
|
if (hasNfa[lexStateIndex] = (NfaState.generatedStates != 0))
|
|
{
|
|
initialState.GenerateCode();
|
|
initialState.GenerateInitMoves(ostr);
|
|
}
|
|
|
|
if (initialState.kind != Integer.MAX_VALUE && initialState.kind != 0)
|
|
{
|
|
if ((toSkip[initialState.kind / 64] & (1L << initialState.kind)) != 0L ||
|
|
(toSpecial[initialState.kind / 64] & (1L << initialState.kind)) != 0L)
|
|
hasSkipActions = true;
|
|
else if ((toMore[initialState.kind / 64] & (1L << initialState.kind)) != 0L)
|
|
hasMoreActions = true;
|
|
else
|
|
hasTokenActions = true;
|
|
|
|
if (initMatch[lexStateIndex] == 0 ||
|
|
initMatch[lexStateIndex] > initialState.kind)
|
|
{
|
|
initMatch[lexStateIndex] = initialState.kind;
|
|
hasEmptyMatch = true;
|
|
}
|
|
}
|
|
else if (initMatch[lexStateIndex] == 0)
|
|
initMatch[lexStateIndex] = Integer.MAX_VALUE;
|
|
|
|
RStringLiteral.FillSubString();
|
|
|
|
if (hasNfa[lexStateIndex] && !mixed[lexStateIndex])
|
|
RStringLiteral.GenerateNfaStartStates(ostr, initialState);
|
|
|
|
RStringLiteral.DumpDfaCode(ostr);
|
|
|
|
if (hasNfa[lexStateIndex])
|
|
NfaState.DumpMoveNfa(ostr);
|
|
|
|
if (stateSetSize < NfaState.generatedStates)
|
|
stateSetSize = NfaState.generatedStates;
|
|
}
|
|
|
|
for (i = 0; i < choices.size(); i++)
|
|
((RChoice)choices.elementAt(i)).CheckUnmatchability();
|
|
|
|
NfaState.DumpStateSets(ostr);
|
|
CheckEmptyStringMatch();
|
|
NfaState.DumpNonAsciiMoveMethods(ostr);
|
|
RStringLiteral.DumpStrLiteralImages(ostr);
|
|
DumpStaticVarDeclarations();
|
|
DumpFillToken();
|
|
DumpGetNextToken();
|
|
|
|
if (Options.getDebugTokenManager())
|
|
{
|
|
NfaState.DumpStatesForKind(ostr);
|
|
DumpDebugMethods();
|
|
}
|
|
|
|
if (hasLoop)
|
|
{
|
|
ostr.println(staticString + "int[] jjemptyLineNo = new int[" + maxLexStates + "];");
|
|
ostr.println(staticString + "int[] jjemptyColNo = new int[" + maxLexStates + "];");
|
|
ostr.println(staticString + "bool[] jjbeenHere = new bool[" + maxLexStates + "];");
|
|
}
|
|
|
|
if (hasSkipActions)
|
|
DumpSkipActions();
|
|
if (hasMoreActions)
|
|
DumpMoreActions();
|
|
if (hasTokenActions)
|
|
DumpTokenActions();
|
|
|
|
ostr.println("}");
|
|
|
|
if(namespace)
|
|
ostr.println("}");
|
|
|
|
ostr.close();
|
|
}
|
|
|
|
static void CheckEmptyStringMatch()
|
|
{
|
|
int i, j, k, len;
|
|
boolean[] seen = new boolean[maxLexStates];
|
|
boolean[] done = new boolean[maxLexStates];
|
|
String cycle;
|
|
String reList;
|
|
|
|
Outer:
|
|
for (i = 0; i < maxLexStates; i++)
|
|
{
|
|
if (done[i] || initMatch[i] == 0 || initMatch[i] == Integer.MAX_VALUE ||
|
|
canMatchAnyChar[i] != -1)
|
|
continue;
|
|
|
|
done[i] = true;
|
|
len = 0;
|
|
cycle = "";
|
|
reList = "";
|
|
|
|
for (k = 0; k < maxLexStates; k++)
|
|
seen[k] = false;
|
|
|
|
j = i;
|
|
seen[i] = true;
|
|
cycle += lexStateName[j] + "-->";
|
|
while (newLexState[initMatch[j]] != null)
|
|
{
|
|
cycle += newLexState[initMatch[j]];
|
|
if (seen[j = GetIndex(newLexState[initMatch[j]])])
|
|
break;
|
|
|
|
cycle += "-->";
|
|
done[j] = true;
|
|
seen[j] = true;
|
|
if (initMatch[j] == 0 || initMatch[j] == Integer.MAX_VALUE ||
|
|
canMatchAnyChar[j] != -1)
|
|
continue Outer;
|
|
if (len != 0)
|
|
reList += "; ";
|
|
reList += "line " + rexprs[initMatch[j]].line + ", column " +
|
|
rexprs[initMatch[j]].column;
|
|
len++;
|
|
}
|
|
|
|
if (newLexState[initMatch[j]] == null)
|
|
cycle += lexStateName[lexStates[initMatch[j]]];
|
|
|
|
for (k = 0; k < maxLexStates; k++)
|
|
canLoop[k] |= seen[k];
|
|
|
|
hasLoop = true;
|
|
if (len == 0)
|
|
CSJavaCCErrors.warning(rexprs[initMatch[i]],
|
|
"Regular expression" + ((rexprs[initMatch[i]].label.equals(""))
|
|
? "" : (" for " + rexprs[initMatch[i]].label)) +
|
|
" can be matched by the empty string (\"\") in lexical state " +
|
|
lexStateName[i] + ". This can result in an endless loop of " +
|
|
"empty string matches.");
|
|
else
|
|
{
|
|
CSJavaCCErrors.warning(rexprs[initMatch[i]],
|
|
"Regular expression" + ((rexprs[initMatch[i]].label.equals(""))
|
|
? "" : (" for " + rexprs[initMatch[i]].label)) +
|
|
" can be matched by the empty string (\"\") in lexical state " +
|
|
lexStateName[i] + ". This regular expression along with the " +
|
|
"regular expressions at " + reList + " forms the cycle \n " +
|
|
cycle + "\ncontaining regular expressions with empty matches." +
|
|
" This can result in an endless loop of empty string matches.");
|
|
}
|
|
}
|
|
}
|
|
|
|
static void PrintArrayInitializer(int noElems)
|
|
{
|
|
ostr.print("{");
|
|
for (int i = 0; i < noElems; i++)
|
|
{
|
|
if (i % 25 == 0)
|
|
ostr.print("\n ");
|
|
ostr.print("0, ");
|
|
}
|
|
ostr.println("\n};");
|
|
}
|
|
|
|
static void DumpStaticVarDeclarations()
|
|
{
|
|
int i;
|
|
String charStreamName;
|
|
|
|
ostr.println("public static String[] lexStateNames = {");
|
|
for (i = 0; i < maxLexStates; i++)
|
|
ostr.println(" \"" + lexStateName[i] + "\", ");
|
|
ostr.println("};");
|
|
|
|
if (maxLexStates > 1){
|
|
ostr.print("public static int[] jjnewLexState = {");
|
|
|
|
for (i = 0; i < maxOrdinal; i++){
|
|
if (i % 25 == 0)
|
|
ostr.print("\n ");
|
|
|
|
if (newLexState[i] == null)
|
|
ostr.print("-1, ");
|
|
else
|
|
ostr.print(GetIndex(newLexState[i]) + ", ");
|
|
}
|
|
ostr.println("\n};");
|
|
}
|
|
|
|
if (hasSkip || hasMore || hasSpecial){
|
|
// Bit vector for TOKEN
|
|
ostr.print("static long[] jjtoToken = {");
|
|
for (i = 0; i < maxOrdinal / 64 + 1; i++){
|
|
if (i % 4 == 0)ostr.print("\n ");
|
|
ostr.print("0x" + Long.toHexString(toToken[i]) + "L, ");
|
|
}
|
|
ostr.println("\n};");
|
|
}
|
|
|
|
if (hasSkip || hasSpecial){
|
|
// Bit vector for SKIP
|
|
ostr.print("static long[] jjtoSkip = {");
|
|
for (i = 0; i < maxOrdinal / 64 + 1; i++){
|
|
if (i % 4 == 0)ostr.print("\n ");
|
|
ostr.print("0x" + Long.toHexString(toSkip[i]) + "L, ");
|
|
}
|
|
ostr.println("\n};");
|
|
}
|
|
|
|
if (hasSpecial)
|
|
{
|
|
// Bit vector for SPECIAL
|
|
ostr.print("static long[] jjtoSpecial = {");
|
|
for (i = 0; i < maxOrdinal / 64 + 1; i++)
|
|
{
|
|
if (i % 4 == 0)
|
|
ostr.print("\n ");
|
|
ostr.print("0x" + Long.toHexString(toSpecial[i]) + "L, ");
|
|
}
|
|
ostr.println("\n};");
|
|
}
|
|
|
|
if (hasMore)
|
|
{
|
|
// Bit vector for MORE
|
|
ostr.print("static long[] jjtoMore = {");
|
|
for (i = 0; i < maxOrdinal / 64 + 1; i++)
|
|
{
|
|
if (i % 4 == 0)
|
|
ostr.print("\n ");
|
|
ostr.print("0x" + Long.toHexString(toMore[i]) + "L, ");
|
|
}
|
|
ostr.println("\n};");
|
|
}
|
|
|
|
if (Options.getUserCharStream())
|
|
charStreamName = "CharStream";
|
|
else
|
|
{
|
|
if (Options.getCSUnicodeEscape())
|
|
charStreamName = "JavaCharStream";
|
|
else
|
|
charStreamName = "SimpleCharStream";
|
|
}
|
|
|
|
ostr.println(staticString + "protected " + charStreamName + " input_stream;");
|
|
|
|
ostr.println(staticString + "private long[] jjrounds = " +
|
|
"new long[" + stateSetSize + "];");
|
|
ostr.println(staticString + "private int[] jjstateSet = " +
|
|
"new int[" + (2 * stateSetSize) + "];");
|
|
|
|
if (hasMoreActions || hasSkipActions || hasTokenActions)
|
|
{
|
|
ostr.println(staticString + "StringBuffer image;");
|
|
ostr.println(staticString + "int jjimageLen;");
|
|
ostr.println(staticString + "int lengthOfMatch;");
|
|
}
|
|
|
|
ostr.println(staticString + "protected char curChar;");
|
|
|
|
if(Options.getTokenManagerUsesParser() && !Options.getStatic()){
|
|
ostr.println("public " + tokMgrClassName + "(" + cu_name + " parserArg, " + charStreamName + " stream){");
|
|
ostr.println(" parser = parserArg;");
|
|
} else {
|
|
ostr.println("public " + tokMgrClassName + "(" + charStreamName + " stream){");
|
|
}
|
|
|
|
if (Options.getStatic() && !Options.getUserCharStream())
|
|
{
|
|
ostr.println(" if (input_stream != null)");
|
|
ostr.println(" throw new TokenMgrError(\"ERROR: Second call to constructor of static lexer. You must use ReInit() to initialize the static variables.\", TokenMgrError.STATIC_LEXER_ERROR);");
|
|
}
|
|
else if (!Options.getUserCharStream())
|
|
{
|
|
if (Options.getCSUnicodeEscape())
|
|
ostr.println(" if (JavaCharStream.staticFlag)");
|
|
else
|
|
ostr.println(" if (SimpleCharStream.staticFlag)");
|
|
|
|
ostr.println(" throw new Exception(\"ERROR: Cannot use a static CharStream class with a non-static lexical analyzer.\");");
|
|
}
|
|
|
|
ostr.println(" input_stream = stream;");
|
|
|
|
ostr.println("}");
|
|
|
|
if(Options.getTokenManagerUsesParser() && !Options.getStatic()){
|
|
ostr.println("public " + tokMgrClassName + "(" + cu_name + " parserArg, " + charStreamName + " stream, int lexState): this(parserArg, stream){");
|
|
} else {
|
|
ostr.println("public " + tokMgrClassName + "(" + charStreamName + " stream, int lexState) : this(stream){");
|
|
}
|
|
ostr.println(" SwitchTo(lexState);");
|
|
ostr.println("}");
|
|
|
|
// Reinit method for reinitializing the parser (for static parsers).
|
|
ostr.println(staticString + "public void ReInit(" + charStreamName + " stream)");
|
|
ostr.println("{");
|
|
ostr.println(" jjmatchedPos = jjnewStateCnt = 0;");
|
|
ostr.println(" curLexState = defaultLexState;");
|
|
ostr.println(" input_stream = stream;");
|
|
ostr.println(" ReInitRounds();");
|
|
ostr.println("}");
|
|
|
|
// Method to reinitialize the jjrounds array.
|
|
ostr.println(staticString + "private void ReInitRounds()");
|
|
ostr.println("{");
|
|
ostr.println(" int i;");
|
|
ostr.println(" jjround = 0x" + Integer.toHexString(Integer.MIN_VALUE + 1)+ ";");
|
|
ostr.println(" for (i = " + stateSetSize + "; i-- > 0;)");
|
|
ostr.println(" jjrounds[i] = 0x" + Integer.toHexString(Integer.MIN_VALUE) + ";");
|
|
ostr.println("}");
|
|
|
|
// Reinit method for reinitializing the parser (for static parsers).
|
|
ostr.println(staticString + "public void ReInit(" + charStreamName + " stream, int lexState)");
|
|
ostr.println("{");
|
|
ostr.println(" ReInit(stream);");
|
|
ostr.println(" SwitchTo(lexState);");
|
|
ostr.println("}");
|
|
|
|
ostr.println(staticString + "public void SwitchTo(int lexState)");
|
|
ostr.println("{");
|
|
ostr.println(" if (lexState >= " + lexStateName.length + " || lexState < 0)");
|
|
ostr.println(" throw new TokenMgrError(\"Error: Ignoring invalid lexical state : \"" +
|
|
" + lexState + \". State unchanged.\", TokenMgrError.INVALID_LEXICAL_STATE);");
|
|
ostr.println(" else");
|
|
ostr.println(" curLexState = lexState;");
|
|
ostr.println("}");
|
|
|
|
ostr.println("");
|
|
}
|
|
|
|
// Assumes l != 0L
|
|
static char MaxChar(long l)
|
|
{
|
|
for (int i = 64; i-- > 0; )
|
|
if ((l & (1L << i)) != 0L)
|
|
return (char)i;
|
|
|
|
return 0xffff;
|
|
}
|
|
|
|
static void DumpFillToken()
|
|
{
|
|
ostr.println(staticString + "protected Token jjFillToken(){");
|
|
ostr.println(" String im = jjstrLiteralImages[jjmatchedKind];");
|
|
ostr.println(" String curTokenIm;");
|
|
if (keepLineCol){
|
|
ostr.println(" int beginLine;");
|
|
ostr.println(" int beginColumn;");
|
|
ostr.println(" int endLine;");
|
|
ostr.println(" int endColumn;");
|
|
}
|
|
if (hasEmptyMatch){
|
|
ostr.println(" if (jjmatchedPos < 0){");
|
|
ostr.println(" if (image == null) curTokenIm = \"\";");
|
|
ostr.println(" else curTokenIm = image.toString();");
|
|
if (keepLineCol){
|
|
ostr.println(" beginLine = endLine = input_stream.getBeginLine();");
|
|
ostr.println(" beginColumn = endColumn = input_stream.getBeginColumn();");
|
|
}
|
|
ostr.println(" }else{");
|
|
}
|
|
ostr.println(" curTokenIm = (im == null) ? input_stream.GetImage() : im;");
|
|
|
|
if (keepLineCol){
|
|
ostr.println(" beginLine = input_stream.getBeginLine();");
|
|
ostr.println(" beginColumn = input_stream.getBeginColumn();");
|
|
ostr.println(" endLine = input_stream.getEndLine();");
|
|
ostr.println(" endColumn = input_stream.getEndColumn();");
|
|
}
|
|
|
|
if (hasEmptyMatch){
|
|
ostr.println(" }");
|
|
}
|
|
ostr.println("");
|
|
ostr.println(" Token t = Token.newToken(jjmatchedKind, curTokenIm);");
|
|
ostr.println(" t.beginLine = beginLine;");
|
|
ostr.println(" t.beginColumn = beginColumn;");
|
|
ostr.println(" t.endLine = endLine;");
|
|
ostr.println(" t.endColumn = endColumn;");
|
|
|
|
ostr.println(" return t;");
|
|
ostr.println("}");
|
|
}
|
|
|
|
static void DumpGetNextToken()
|
|
{
|
|
int i;
|
|
|
|
ostr.println("");
|
|
ostr.println(staticString + "int curLexState = " + defaultLexState + ";");
|
|
ostr.println(staticString + "int defaultLexState = " + defaultLexState + ";");
|
|
ostr.println(staticString + "int jjnewStateCnt;");
|
|
ostr.println(staticString + "long jjround;");
|
|
ostr.println(staticString + "int jjmatchedPos;");
|
|
ostr.println(staticString + "int jjmatchedKind;");
|
|
ostr.println("");
|
|
ostr.println("public " + staticString + "Token getNextToken()" +
|
|
" ");
|
|
ostr.println("{");
|
|
ostr.println(" int kind;");
|
|
ostr.println(" Token specialToken = null;");
|
|
ostr.println(" Token matchedToken;");
|
|
ostr.println(" int curPos = 0;");
|
|
ostr.println("");
|
|
ostr.println(" EOFLoop :\n for (;;){");
|
|
ostr.println(" try{ ");
|
|
ostr.println(" curChar = input_stream.BeginToken();");
|
|
ostr.println(" }catch(System.IO.IOException e){");
|
|
|
|
if (Options.getDebugTokenManager())
|
|
ostr.println(" debugStream.WriteLine(\"Returning the <EOF> token.\");");
|
|
|
|
ostr.println(" jjmatchedKind = 0;");
|
|
ostr.println(" matchedToken = jjFillToken();");
|
|
|
|
if (hasSpecial)
|
|
ostr.println(" matchedToken.specialToken = specialToken;");
|
|
|
|
if (nextStateForEof != null || actForEof != null)
|
|
ostr.println(" TokenLexicalActions(matchedToken);");
|
|
|
|
if (Options.getCommonTokenAction())
|
|
ostr.println(" CommonTokenAction(matchedToken);");
|
|
|
|
ostr.println(" return matchedToken;");
|
|
ostr.println(" }");
|
|
|
|
if (hasMoreActions || hasSkipActions || hasTokenActions){
|
|
ostr.println(" image = null;");
|
|
ostr.println(" jjimageLen = 0;");
|
|
}
|
|
|
|
ostr.println("");
|
|
|
|
String prefix = "";
|
|
if (hasMore){
|
|
ostr.println(" for (;;){");
|
|
prefix = " ";
|
|
}
|
|
|
|
String endSwitch = "";
|
|
String caseStr = "";
|
|
// this also sets up the start state of the nfa
|
|
if (maxLexStates > 1){
|
|
ostr.println(prefix + " switch(curLexState){");
|
|
endSwitch = prefix + " }";
|
|
caseStr = prefix + " case ";
|
|
prefix += " ";
|
|
}
|
|
|
|
prefix += " ";
|
|
for(i = 0; i < maxLexStates; i++){
|
|
if (maxLexStates > 1)
|
|
ostr.println(caseStr + i + ":");
|
|
|
|
if (singlesToSkip[i].HasTransitions())
|
|
{
|
|
// added the backup(0) to make JIT happy
|
|
ostr.println(prefix + "try { input_stream.backup(0);");
|
|
if (singlesToSkip[i].asciiMoves[0] != 0L &&
|
|
singlesToSkip[i].asciiMoves[1] != 0L){
|
|
ostr.println(prefix + " while ((curChar < 64" + " && (0x" +
|
|
Long.toHexString(singlesToSkip[i].asciiMoves[0]) +
|
|
"L & (1L << curChar)) != 0L) || \n" +
|
|
prefix + " (curChar >> 6) == 1" +
|
|
" && (0x" +
|
|
Long.toHexString(singlesToSkip[i].asciiMoves[1]) +
|
|
"L & (1L << (curChar & 0x3F))) != 0L)");
|
|
}
|
|
else if (singlesToSkip[i].asciiMoves[1] == 0L)
|
|
{
|
|
ostr.println(prefix + " while (curChar <= " +
|
|
(int)MaxChar(singlesToSkip[i].asciiMoves[0]) + " && (0x" +
|
|
Long.toHexString(singlesToSkip[i].asciiMoves[0]) +
|
|
"L & (1L << curChar)) != 0L)");
|
|
}
|
|
else if (singlesToSkip[i].asciiMoves[0] == 0L)
|
|
{
|
|
ostr.println(prefix + " while (curChar > 63 && curChar <= " +
|
|
((int)MaxChar(singlesToSkip[i].asciiMoves[1]) + 64) +
|
|
" && (0x" +
|
|
Long.toHexString(singlesToSkip[i].asciiMoves[1]) +
|
|
"L & (1L << (curChar & 0x3F))) != 0L)");
|
|
}
|
|
|
|
if (Options.getDebugTokenManager())
|
|
{
|
|
ostr.println(prefix + "{");
|
|
ostr.println(" debugStream.WriteLine(" + (maxLexStates > 1 ? "\"<\" + lexStateNames[curLexState]\n + \">\" + " : "") + "\"Skipping character : \" + " +
|
|
"TokenMgrError.addEscapes(\"\"+(curChar)) + \" (\" + (int)curChar + \")\");");
|
|
}
|
|
ostr.println(prefix + " curChar = input_stream.BeginToken();");
|
|
|
|
if (Options.getDebugTokenManager())
|
|
ostr.println(prefix + "}");
|
|
|
|
ostr.println(prefix + "}");
|
|
ostr.println(prefix + "catch (System.IO.IOException e1) { goto EOFLoop; }");
|
|
}
|
|
|
|
if (initMatch[i] != Integer.MAX_VALUE && initMatch[i] != 0)
|
|
{
|
|
if (Options.getDebugTokenManager())
|
|
ostr.println(" debugStream.WriteLine(\" Matched the empty string as \" + tokenImage[" +
|
|
initMatch[i] + "] + \" token.\");");
|
|
|
|
ostr.println(prefix + "jjmatchedKind = " + initMatch[i] + ";");
|
|
ostr.println(prefix + "jjmatchedPos = -1;");
|
|
ostr.println(prefix + "curPos = 0;");
|
|
}
|
|
else
|
|
{
|
|
ostr.println(prefix + "jjmatchedKind = 0x" + Integer.toHexString(Integer.MAX_VALUE) + ";");
|
|
ostr.println(prefix + "jjmatchedPos = 0;");
|
|
}
|
|
|
|
if (Options.getDebugTokenManager())
|
|
ostr.println(" debugStream.WriteLine(" + (maxLexStates > 1 ? "\"<\" + lexStateNames[curLexState] + \">\" + " : "") + "\"Current character : \" + " +
|
|
"TokenMgrError.addEscapes(\"\"+(curChar)) + \" (\" + (int)curChar + \") at line \" + input_stream.getLine() + \" column \" + input_stream.getColumn());");
|
|
|
|
ostr.println(prefix + "curPos = jjMoveStringLiteralDfa0_" + i + "();");
|
|
|
|
if (canMatchAnyChar[i] != -1)
|
|
{
|
|
if (initMatch[i] != Integer.MAX_VALUE && initMatch[i] != 0)
|
|
ostr.println(prefix + "if (jjmatchedPos < 0 || (jjmatchedPos == 0 && jjmatchedKind > " +
|
|
canMatchAnyChar[i] + "))");
|
|
else
|
|
ostr.println(prefix + "if (jjmatchedPos == 0 && jjmatchedKind > " +
|
|
canMatchAnyChar[i] + ")");
|
|
ostr.println(prefix + "{");
|
|
|
|
if (Options.getDebugTokenManager())
|
|
ostr.println(" debugStream.WriteLine(\" Current character matched as a \" + tokenImage[" +
|
|
canMatchAnyChar[i] + "] + \" token.\");");
|
|
ostr.println(prefix + " jjmatchedKind = " + canMatchAnyChar[i] + ";");
|
|
|
|
if (initMatch[i] != Integer.MAX_VALUE && initMatch[i] != 0)
|
|
ostr.println(prefix + " jjmatchedPos = 0;");
|
|
|
|
ostr.println(prefix + "}");
|
|
}
|
|
|
|
if (maxLexStates > 1)
|
|
ostr.println(prefix + "break;");
|
|
}
|
|
|
|
if (maxLexStates > 1)
|
|
ostr.println(endSwitch);
|
|
else if (maxLexStates == 0)
|
|
ostr.println(" jjmatchedKind = 0x" + Integer.toHexString(Integer.MAX_VALUE) + ";");
|
|
|
|
if (maxLexStates > 1)prefix = " ";
|
|
else prefix = "";
|
|
|
|
if (maxLexStates > 0){
|
|
ostr.println(prefix + " if (jjmatchedKind != 0x" + Integer.toHexString(Integer.MAX_VALUE) + "){");
|
|
ostr.println(prefix + " if (jjmatchedPos + 1 < curPos)");
|
|
|
|
if (Options.getDebugTokenManager())
|
|
ostr.println(prefix + " { debugStream.WriteLine(\" Putting back \" + (curPos - jjmatchedPos - 1) + \" characters into the input stream.\");");
|
|
|
|
ostr.println(prefix + " input_stream.backup(curPos - jjmatchedPos - 1);");
|
|
|
|
if (Options.getDebugTokenManager())
|
|
ostr.println(prefix + " }");
|
|
|
|
if (Options.getDebugTokenManager())
|
|
ostr.println(" debugStream.WriteLine(\"****** FOUND A \" + tokenImage[jjmatchedKind] + \" MATCH (\" + TokenMgrError.addEscapes(new String(input_stream.GetSuffix(jjmatchedPos + 1))) + \") ******\\n\");");
|
|
|
|
if (hasSkip || hasMore || hasSpecial){
|
|
ostr.println(prefix + " if (((ulong)jjtoToken[jjmatchedKind >> 6] & (1UL << (jjmatchedKind & 0x3F))) != 0){");
|
|
}
|
|
|
|
ostr.println(prefix + " matchedToken = jjFillToken();");
|
|
|
|
if (hasSpecial)
|
|
ostr.println(prefix + " matchedToken.specialToken = specialToken;");
|
|
|
|
if (hasTokenActions)
|
|
ostr.println(prefix + " TokenLexicalActions(matchedToken);");
|
|
|
|
if (maxLexStates > 1){
|
|
ostr.println(" if (jjnewLexState[jjmatchedKind] != -1)");
|
|
ostr.println(prefix + " curLexState = jjnewLexState[jjmatchedKind];");
|
|
}
|
|
|
|
if (Options.getCommonTokenAction())
|
|
ostr.println(prefix + " CommonTokenAction(matchedToken);");
|
|
|
|
ostr.println(prefix + " return matchedToken;");
|
|
|
|
if (hasSkip || hasMore || hasSpecial)
|
|
{
|
|
ostr.println(prefix + " }");
|
|
|
|
if (hasSkip || hasSpecial)
|
|
{
|
|
if (hasMore)
|
|
{
|
|
ostr.println(prefix + " else if ((jjtoSkip[jjmatchedKind >> 6] & " +
|
|
"(1L << (jjmatchedKind & 0x3F))) != 0L)");
|
|
}
|
|
else
|
|
ostr.println(prefix + " else");
|
|
|
|
ostr.println(prefix + " {");
|
|
|
|
if (hasSpecial){
|
|
ostr.println(prefix + " if ((jjtoSpecial[jjmatchedKind >> 6] & " +
|
|
"(1L << (jjmatchedKind & 0x3F))) != 0L)");
|
|
ostr.println(prefix + " {");
|
|
|
|
ostr.println(prefix + " matchedToken = jjFillToken();");
|
|
|
|
ostr.println(prefix + " if (specialToken == null)");
|
|
ostr.println(prefix + " specialToken = matchedToken;");
|
|
ostr.println(prefix + " else");
|
|
ostr.println(prefix + " {");
|
|
ostr.println(prefix + " matchedToken.specialToken = specialToken;");
|
|
ostr.println(prefix + " specialToken = (specialToken.next = matchedToken);");
|
|
ostr.println(prefix + " }");
|
|
|
|
if (hasSkipActions)
|
|
ostr.println(prefix + " SkipLexicalActions(matchedToken);");
|
|
|
|
ostr.println(prefix + " }");
|
|
|
|
if (hasSkipActions)
|
|
{
|
|
ostr.println(prefix + " else ");
|
|
ostr.println(prefix + " SkipLexicalActions(null);");
|
|
}
|
|
}
|
|
else if (hasSkipActions)
|
|
ostr.println(prefix + " SkipLexicalActions(null);");
|
|
|
|
if (maxLexStates > 1)
|
|
{
|
|
ostr.println(" if (jjnewLexState[jjmatchedKind] != -1)");
|
|
ostr.println(prefix + " curLexState = jjnewLexState[jjmatchedKind];");
|
|
}
|
|
|
|
ostr.println(prefix + " goto EOFLoop;");
|
|
ostr.println(prefix + " }");
|
|
}
|
|
|
|
if (hasMore)
|
|
{
|
|
if (hasMoreActions)
|
|
ostr.println(prefix + " MoreLexicalActions();");
|
|
else if (hasSkipActions || hasTokenActions)
|
|
ostr.println(prefix + " jjimageLen += jjmatchedPos + 1;");
|
|
|
|
if (maxLexStates > 1)
|
|
{
|
|
ostr.println(" if (jjnewLexState[jjmatchedKind] != -1)");
|
|
ostr.println(prefix + " curLexState = jjnewLexState[jjmatchedKind];");
|
|
}
|
|
ostr.println(prefix + " curPos = 0;");
|
|
ostr.println(prefix + " jjmatchedKind = 0x" + Integer.toHexString(Integer.MAX_VALUE) + ";");
|
|
|
|
ostr.println(prefix + " try {");
|
|
ostr.println(prefix + " curChar = input_stream.readChar();");
|
|
|
|
if (Options.getDebugTokenManager())
|
|
ostr.println(" debugStream.WriteLine(" + (maxLexStates > 1 ? "\"<\" + lexStateNames[curLexState] + \">\" + " : "") + "\"Current character : \" + " +
|
|
"TokenMgrError.addEscapes(\"\"+(curChar)) + \" (\" + (int)curChar + \") at line \" + input_stream.getLine() + \" column \" + input_stream.getColumn());");
|
|
ostr.println(prefix + " continue;");
|
|
ostr.println(prefix + " }");
|
|
ostr.println(prefix + " catch (System.IO.IOException e1) { }");
|
|
}
|
|
}
|
|
|
|
ostr.println(prefix + " }");
|
|
ostr.println(prefix + " int error_line = input_stream.getEndLine();");
|
|
ostr.println(prefix + " int error_column = input_stream.getEndColumn();");
|
|
ostr.println(prefix + " String error_after = null;");
|
|
ostr.println(prefix + " bool EOFSeen = false;");
|
|
ostr.println(prefix + " try { input_stream.readChar(); input_stream.backup(1); }");
|
|
ostr.println(prefix + " catch (System.IO.IOException e1) {");
|
|
ostr.println(prefix + " EOFSeen = true;");
|
|
ostr.println(prefix + " error_after = curPos <= 1 ? \"\" : input_stream.GetImage();");
|
|
ostr.println(prefix + " if (curChar == '\\n' || curChar == '\\r') {");
|
|
ostr.println(prefix + " error_line++;");
|
|
ostr.println(prefix + " error_column = 0;");
|
|
ostr.println(prefix + " }");
|
|
ostr.println(prefix + " else");
|
|
ostr.println(prefix + " error_column++;");
|
|
ostr.println(prefix + " }");
|
|
ostr.println(prefix + " if (!EOFSeen) {");
|
|
ostr.println(prefix + " input_stream.backup(1);");
|
|
ostr.println(prefix + " error_after = curPos <= 1 ? \"\" : input_stream.GetImage();");
|
|
ostr.println(prefix + " }");
|
|
ostr.println(prefix + " throw new TokenMgrError(EOFSeen, curLexState, error_line, error_column, error_after, curChar, TokenMgrError.LEXICAL_ERROR);");
|
|
}
|
|
|
|
if (hasMore)
|
|
ostr.println(prefix + " }");
|
|
|
|
ostr.println(" }");
|
|
ostr.println("}");
|
|
ostr.println("");
|
|
}
|
|
|
|
public static void DumpSkipActions()
|
|
{
|
|
Action act;
|
|
|
|
ostr.println(staticString + "void SkipLexicalActions(Token matchedToken)");
|
|
ostr.println("{");
|
|
ostr.println(" switch(jjmatchedKind)");
|
|
ostr.println(" {");
|
|
|
|
Outer:
|
|
for (int i = 0; i < maxOrdinal; i++)
|
|
{
|
|
if ((toSkip[i / 64] & (1L << (i % 64))) == 0L)
|
|
continue;
|
|
|
|
for (;;)
|
|
{
|
|
if (((act = (Action)actions[i]) == null ||
|
|
act.action_tokens == null ||
|
|
act.action_tokens.size() == 0) && !canLoop[lexStates[i]])
|
|
continue Outer;
|
|
|
|
ostr.println(" case " + i + " :");
|
|
|
|
if (initMatch[lexStates[i]] == i && canLoop[lexStates[i]])
|
|
{
|
|
ostr.println(" if (jjmatchedPos == -1)");
|
|
ostr.println(" {");
|
|
ostr.println(" if (jjbeenHere[" + lexStates[i] + "] &&");
|
|
ostr.println(" jjemptyLineNo[" + lexStates[i] + "] == input_stream.getBeginLine() && ");
|
|
ostr.println(" jjemptyColNo[" + lexStates[i] + "] == input_stream.getBeginColumn())");
|
|
ostr.println(" throw new TokenMgrError((\"Error: Bailing out of infinite loop caused by repeated empty string matches at line \" + input_stream.getBeginLine() + \", column \" + input_stream.getBeginColumn() + \".\"), TokenMgrError.LOOP_DETECTED);");
|
|
ostr.println(" jjemptyLineNo[" + lexStates[i] + "] = input_stream.getBeginLine();");
|
|
ostr.println(" jjemptyColNo[" + lexStates[i] + "] = input_stream.getBeginColumn();");
|
|
ostr.println(" jjbeenHere[" + lexStates[i] + "] = true;");
|
|
ostr.println(" }");
|
|
}
|
|
|
|
if ((act = (Action)actions[i]) == null ||
|
|
act.action_tokens.size() == 0)
|
|
break;
|
|
|
|
ostr.println(" if (image == null)");
|
|
ostr.println(" image = new StringBuffer();");
|
|
|
|
ostr.print(" image.append");
|
|
if (RStringLiteral.allImages[i] != null)
|
|
ostr.println("(jjstrLiteralImages[" + i + "]);");
|
|
else
|
|
if (Options.getCSUnicodeEscape() ||
|
|
Options.getUserCharStream())
|
|
ostr.println("(input_stream.GetSuffix(jjimageLen + (lengthOfMatch = jjmatchedPos + 1)));");
|
|
else
|
|
ostr.println("(input_stream.GetSuffix(jjimageLen + (lengthOfMatch = jjmatchedPos + 1)));");
|
|
|
|
printTokenSetup((Token)act.action_tokens.elementAt(0));
|
|
ccol = 1;
|
|
|
|
for (int j = 0; j < act.action_tokens.size(); j++)
|
|
printToken((Token)act.action_tokens.elementAt(j), ostr);
|
|
ostr.println("");
|
|
|
|
break;
|
|
}
|
|
|
|
ostr.println(" break;");
|
|
}
|
|
|
|
ostr.println(" default :");
|
|
ostr.println(" break;");
|
|
ostr.println(" }");
|
|
ostr.println("}");
|
|
}
|
|
|
|
public static void DumpMoreActions()
|
|
{
|
|
Action act;
|
|
|
|
ostr.println(staticString + "void MoreLexicalActions()");
|
|
ostr.println("{");
|
|
ostr.println(" jjimageLen += (lengthOfMatch = jjmatchedPos + 1);");
|
|
ostr.println(" switch(jjmatchedKind)");
|
|
ostr.println(" {");
|
|
|
|
Outer:
|
|
for (int i = 0; i < maxOrdinal; i++)
|
|
{
|
|
if ((toMore[i / 64] & (1L << (i % 64))) == 0L)
|
|
continue;
|
|
|
|
for (;;)
|
|
{
|
|
if (((act = (Action)actions[i]) == null ||
|
|
act.action_tokens == null ||
|
|
act.action_tokens.size() == 0) && !canLoop[lexStates[i]])
|
|
continue Outer;
|
|
|
|
ostr.println(" case " + i + " :");
|
|
|
|
if (initMatch[lexStates[i]] == i && canLoop[lexStates[i]])
|
|
{
|
|
ostr.println(" if (jjmatchedPos == -1)");
|
|
ostr.println(" {");
|
|
ostr.println(" if (jjbeenHere[" + lexStates[i] + "] &&");
|
|
ostr.println(" jjemptyLineNo[" + lexStates[i] + "] == input_stream.getBeginLine() && ");
|
|
ostr.println(" jjemptyColNo[" + lexStates[i] + "] == input_stream.getBeginColumn())");
|
|
ostr.println(" throw new TokenMgrError((\"Error: Bailing out of infinite loop caused by repeated empty string matches at line \" + input_stream.getBeginLine() + \", column \" + input_stream.getBeginColumn() + \".\"), TokenMgrError.LOOP_DETECTED);");
|
|
ostr.println(" jjemptyLineNo[" + lexStates[i] + "] = input_stream.getBeginLine();");
|
|
ostr.println(" jjemptyColNo[" + lexStates[i] + "] = input_stream.getBeginColumn();");
|
|
ostr.println(" jjbeenHere[" + lexStates[i] + "] = true;");
|
|
ostr.println(" }");
|
|
}
|
|
|
|
if ((act = (Action)actions[i]) == null ||
|
|
act.action_tokens.size() == 0)
|
|
{
|
|
break;
|
|
}
|
|
|
|
ostr.println(" if (image == null)");
|
|
ostr.println(" image = new StringBuffer();");
|
|
ostr.print(" image.append");
|
|
|
|
if (RStringLiteral.allImages[i] != null)
|
|
ostr.println("(jjstrLiteralImages[" + i + "]);");
|
|
else
|
|
if (Options.getCSUnicodeEscape() ||
|
|
Options.getUserCharStream())
|
|
ostr.println("(input_stream.GetSuffix(jjimageLen));");
|
|
else
|
|
ostr.println("(input_stream.GetSuffix(jjimageLen));");
|
|
|
|
ostr.println(" jjimageLen = 0;");
|
|
printTokenSetup((Token)act.action_tokens.elementAt(0));
|
|
ccol = 1;
|
|
|
|
for (int j = 0; j < act.action_tokens.size(); j++)
|
|
printToken((Token)act.action_tokens.elementAt(j), ostr);
|
|
ostr.println("");
|
|
|
|
break;
|
|
}
|
|
|
|
ostr.println(" break;");
|
|
}
|
|
|
|
ostr.println(" default : ");
|
|
ostr.println(" break;");
|
|
|
|
ostr.println(" }");
|
|
ostr.println("}");
|
|
}
|
|
|
|
public static void DumpTokenActions()
|
|
{
|
|
Action act;
|
|
int i;
|
|
|
|
ostr.println(staticString + "void TokenLexicalActions(Token matchedToken)");
|
|
ostr.println("{");
|
|
ostr.println(" switch(jjmatchedKind)");
|
|
ostr.println(" {");
|
|
|
|
Outer:
|
|
for (i = 0; i < maxOrdinal; i++)
|
|
{
|
|
if ((toToken[i / 64] & (1L << (i % 64))) == 0L)
|
|
continue;
|
|
|
|
for (;;)
|
|
{
|
|
if (((act = (Action)actions[i]) == null ||
|
|
act.action_tokens == null ||
|
|
act.action_tokens.size() == 0) && !canLoop[lexStates[i]])
|
|
continue Outer;
|
|
|
|
ostr.println(" case " + i + " :");
|
|
|
|
if (initMatch[lexStates[i]] == i && canLoop[lexStates[i]])
|
|
{
|
|
ostr.println(" if (jjmatchedPos == -1)");
|
|
ostr.println(" {");
|
|
ostr.println(" if (jjbeenHere[" + lexStates[i] + "] &&");
|
|
ostr.println(" jjemptyLineNo[" + lexStates[i] + "] == input_stream.getBeginLine() && ");
|
|
ostr.println(" jjemptyColNo[" + lexStates[i] + "] == input_stream.getBeginColumn())");
|
|
ostr.println(" throw new TokenMgrError((\"Error: Bailing out of infinite loop caused by repeated empty string matches at line \" + input_stream.getBeginLine() + \", column \" + input_stream.getBeginColumn() + \".\"), TokenMgrError.LOOP_DETECTED);");
|
|
ostr.println(" jjemptyLineNo[" + lexStates[i] + "] = input_stream.getBeginLine();");
|
|
ostr.println(" jjemptyColNo[" + lexStates[i] + "] = input_stream.getBeginColumn();");
|
|
ostr.println(" jjbeenHere[" + lexStates[i] + "] = true;");
|
|
ostr.println(" }");
|
|
}
|
|
|
|
if ((act = (Action)actions[i]) == null ||
|
|
act.action_tokens.size() == 0)
|
|
break;
|
|
|
|
if (i == 0)
|
|
{
|
|
ostr.println(" image = null;"); // For EOF no image is there
|
|
}
|
|
else
|
|
{
|
|
ostr.println(" if (image == null)");
|
|
ostr.println(" image = new StringBuffer();");
|
|
ostr.print(" image.append");
|
|
|
|
if (RStringLiteral.allImages[i] != null)
|
|
ostr.println("(jjstrLiteralImages[" + i + "]);");
|
|
else
|
|
if (Options.getCSUnicodeEscape() ||
|
|
Options.getUserCharStream())
|
|
ostr.println("(input_stream.GetSuffix(jjimageLen + (lengthOfMatch = jjmatchedPos + 1)));");
|
|
else
|
|
ostr.println("(input_stream.GetSuffix(jjimageLen + (lengthOfMatch = jjmatchedPos + 1)));");
|
|
|
|
}
|
|
printTokenSetup((Token)act.action_tokens.elementAt(0));
|
|
ccol = 1;
|
|
|
|
for (int j = 0; j < act.action_tokens.size(); j++)
|
|
printToken((Token)act.action_tokens.elementAt(j), ostr);
|
|
ostr.println("");
|
|
|
|
break;
|
|
}
|
|
|
|
ostr.println(" break;");
|
|
}
|
|
|
|
ostr.println(" default : ");
|
|
ostr.println(" break;");
|
|
ostr.println(" }");
|
|
ostr.println("}");
|
|
}
|
|
|
|
public static void reInit()
|
|
{
|
|
ostr = null;
|
|
staticString = null;
|
|
tokMgrClassName = null;
|
|
allTpsForState = new Hashtable();
|
|
lexStateIndex = 0;
|
|
kinds = null;
|
|
maxOrdinal = 1;
|
|
lexStateSuffix = null;
|
|
newLexState = null;
|
|
lexStates = null;
|
|
ignoreCase = null;
|
|
actions = null;
|
|
initStates = new Hashtable();
|
|
stateSetSize = 0;
|
|
maxLexStates = 0;
|
|
lexStateName = null;
|
|
singlesToSkip = null;
|
|
toSkip = null;
|
|
toSpecial = null;
|
|
toMore = null;
|
|
toToken = null;
|
|
defaultLexState = 0;
|
|
rexprs = null;
|
|
maxLongsReqd = null;
|
|
initMatch = null;
|
|
canMatchAnyChar = null;
|
|
hasEmptyMatch = false;
|
|
canLoop = null;
|
|
stateHasActions = null;
|
|
hasLoop = false;
|
|
canReachOnMore = null;
|
|
hasNfa = null;
|
|
mixed = null;
|
|
initialState = null;
|
|
curKind = 0;
|
|
hasSkipActions = false;
|
|
hasMoreActions = false;
|
|
hasTokenActions = false;
|
|
hasSpecial = false;
|
|
hasSkip = false;
|
|
hasMore = false;
|
|
hasToken = false;
|
|
curRE = null;
|
|
}
|
|
|
|
}
|