Module grammar

Package de.grogra.grammar

Class Tokenizer

java.lang.Object

de.grogra.grammar.Tokenizer

All Implemented Interfaces:

antlr.TokenStream

Direct Known Subclasses:

DTDTokenizer, JavaTokenizer, LSYTokenizer, XLTokenizer

public class Tokenizer
extends Object
implements antlr.TokenStream

Field Summary

Fields

Modifier and Type	Field	Description
static final int	CREATE_TOKEN_LOCATOR
protected final boolean	createTokenLocator
static final int	ENABLE_TYPE_SUFFIX
protected final boolean	enableTypeSuffix
static final int	EOL_IS_SIGNIFICANT
protected final boolean	eolIsSignificant
static final int	EVALUATE_NUMBERS
protected final boolean	evaluateNumbers
static final int	FLOAT_IS_DEFAULT
protected final boolean	floatIsDefault
static final I18NBundle	I18N
static final int	MINUS_IS_SIGN
protected final boolean	minusIsSign
protected final boolean	unicode
static final int	UNICODE_ESCAPES

Constructor Summary

Constructors

Constructor	Description
Tokenizer(int flags)

Method Summary

Modifier and Type	Method	Description
final void	addToken(int tokenId, String string)
final void	addToken(int tokenId, String string, boolean identifierPrecedes)
final void	addToken(Token token)
final void	addToken(Token token, boolean identifierPrecedes)
final void	addToken(Token token, String string)
final void	addToken(Token token, String string, boolean identifierPrecedes)
final void	consume(int type)
protected Token	convert(Token t)
protected void	copyTokenTablesFrom(Tokenizer source)
final float	getFloat()
final Input	getInput()
final int	getInt()
final NumberLiteral	getNumberToken(int allowedLiterals)
final Token	getToken()
protected boolean	isIdentifierPart(char c)
protected boolean	isIdentifierStart(char c)
protected boolean	isWhitespace(char c)
antlr.Token	nextToken()
final void	reset()
final void	setInput(Input input)
final void	setSource(Reader source)
final void	setSource(Reader source, String name)
final void	ungetToken()

Methods inherited from class java.lang.Object

clone, equals, finalize, getClass, hashCode, notify, notifyAll, toString, wait, wait, wait

Field Details

I18N

public static final I18NBundle I18N

EOL_IS_SIGNIFICANT

public static final int EOL_IS_SIGNIFICANT

See Also:

• Constant Field Values

FLOAT_IS_DEFAULT

public static final int FLOAT_IS_DEFAULT

See Also:

• Constant Field Values

MINUS_IS_SIGN

public static final int MINUS_IS_SIGN

See Also:

• Constant Field Values

CREATE_TOKEN_LOCATOR

public static final int CREATE_TOKEN_LOCATOR

See Also:

• Constant Field Values

EVALUATE_NUMBERS

public static final int EVALUATE_NUMBERS

See Also:

• Constant Field Values

ENABLE_TYPE_SUFFIX

public static final int ENABLE_TYPE_SUFFIX

See Also:

• Constant Field Values

UNICODE_ESCAPES

public static final int UNICODE_ESCAPES

See Also:

• Constant Field Values

eolIsSignificant

protected final boolean eolIsSignificant

floatIsDefault

protected final boolean floatIsDefault

minusIsSign

protected final boolean minusIsSign

createTokenLocator

protected final boolean createTokenLocator

evaluateNumbers

protected final boolean evaluateNumbers

enableTypeSuffix

protected final boolean enableTypeSuffix

unicode

protected final boolean unicode

Constructor Details

Tokenizer

public Tokenizer(int flags)

Method Details

copyTokenTablesFrom

protected void copyTokenTablesFrom(Tokenizer source)

addToken

public final void addToken(Token token)

addToken

public final void addToken(Token token,
 boolean identifierPrecedes)

addToken

public final void addToken(int tokenId,
 String string)

addToken

public final void addToken(int tokenId,
 String string,
 boolean identifierPrecedes)

addToken

public final void addToken(Token token,
 String string)

addToken

public final void addToken(Token token,
 String string,
 boolean identifierPrecedes)

setInput

public final void setInput(Input input)

setSource

public final void setSource(Reader source)

setSource

public final void setSource(Reader source,
 String name)

getInput

public final Input getInput()

reset

public final void reset()

isWhitespace

protected boolean isWhitespace(char c)

isIdentifierStart

protected boolean isIdentifierStart(char c)

isIdentifierPart

protected boolean isIdentifierPart(char c)

convert

protected Token convert(Token t)

getToken

public final Token getToken()
                     throws IOException,
LexicalException

Throws:

IOException

LexicalException

nextToken

public antlr.Token nextToken()
                      throws antlr.TokenStreamException

Specified by:

nextToken in interface antlr.TokenStream

Throws:

antlr.TokenStreamException

ungetToken

public final void ungetToken()

getNumberToken

public final NumberLiteral getNumberToken(int allowedLiterals)
                                   throws IOException,
LexicalException

Throws:

IOException

LexicalException

getInt

public final int getInt()
                 throws IOException,
LexicalException

Throws:

IOException

LexicalException

getFloat

public final float getFloat()
                     throws IOException,
LexicalException

Throws:

IOException

LexicalException

consume

public final void consume(int type)
                   throws IOException,
LexicalException,
UnexpectedTokenException

Throws:

IOException

LexicalException

UnexpectedTokenException