Class TokenStream

java.lang.Object
org.mozilla.javascript.TokenStream

class TokenStream extends Object
This class implements the JavaScript scanner.

It is based on the C source files jsscan.c and jsscan.h in the jsref package.

See Also:
  • Field Details

    • EOF_CHAR

      private static final int EOF_CHAR
      See Also:
    • REPORT_NUMBER_FORMAT_ERROR

      private static final int REPORT_NUMBER_FORMAT_ERROR
      See Also:
    • BYTE_ORDER_MARK

      private static final char BYTE_ORDER_MARK
      See Also:
    • NUMERIC_SEPARATOR

      private static final char NUMERIC_SEPARATOR
      See Also:
    • rawString

      private StringBuilder rawString
    • dirtyLine

      private boolean dirtyLine
    • regExpFlags

      String regExpFlags
    • string

      private String string
    • number

      private double number
    • bigInt

      private BigInteger bigInt
    • isBinary

      private boolean isBinary
    • isOldOctal

      private boolean isOldOctal
    • isOctal

      private boolean isOctal
    • isHex

      private boolean isHex
    • quoteChar

      private int quoteChar
    • stringBuffer

      private char[] stringBuffer
    • stringBufferTop

      private int stringBufferTop
    • allStrings

      private ObjToIntMap allStrings
    • ungetBuffer

      private final int[] ungetBuffer
    • ungetCursor

      private int ungetCursor
    • hitEOF

      private boolean hitEOF
    • lineStart

      private int lineStart
    • lineEndChar

      private int lineEndChar
    • lineno

      int lineno
    • sourceString

      private String sourceString
    • sourceReader

      private Reader sourceReader
    • sourceBuffer

      private char[] sourceBuffer
    • sourceEnd

      private int sourceEnd
    • sourceCursor

      int sourceCursor
    • cursor

      int cursor
    • tokenBeg

      int tokenBeg
    • tokenEnd

      int tokenEnd
    • commentType

      Token.CommentType commentType
    • xmlIsAttribute

      private boolean xmlIsAttribute
    • xmlIsTagContent

      private boolean xmlIsTagContent
    • xmlOpenTagsCount

      private int xmlOpenTagsCount
    • parser

      private Parser parser
    • commentPrefix

      private String commentPrefix
    • commentCursor

      private int commentCursor
  • Constructor Details

    • TokenStream

      TokenStream(Parser parser, Reader sourceReader, String sourceString, int lineno)
  • Method Details

    • tokenToString

      String tokenToString(int token)
    • isKeyword

      static boolean isKeyword(String s, int version, boolean isStrict)
    • stringToKeyword

      private static int stringToKeyword(String name, int version, boolean isStrict)
    • stringToKeywordForJS

      private static int stringToKeywordForJS(String name)
      JavaScript 1.8 and earlier
    • stringToKeywordForES

      private static int stringToKeywordForES(String name, boolean isStrict)
      ECMAScript 6.
    • getSourceString

      final String getSourceString()
    • getLineno

      final int getLineno()
    • getString

      final String getString()
    • getQuoteChar

      final char getQuoteChar()
    • getNumber

      final double getNumber()
    • getBigInt

      final BigInteger getBigInt()
    • isNumericBinary

      final boolean isNumericBinary()
    • isNumericOldOctal

      final boolean isNumericOldOctal()
    • isNumericOctal

      final boolean isNumericOctal()
    • isNumericHex

      final boolean isNumericHex()
    • eof

      final boolean eof()
    • getToken

      final int getToken() throws IOException
      Throws:
      IOException
    • readDigits

      private int readDigits(int base, int c) throws IOException
      Throws:
      IOException
    • isAlpha

      private static boolean isAlpha(int c)
    • isDigit

      private static boolean isDigit(int base, int c)
    • isDualDigit

      private static boolean isDualDigit(int c)
    • isOctalDigit

      private static boolean isOctalDigit(int c)
    • isDigit

      private static boolean isDigit(int c)
    • isHexDigit

      private static boolean isHexDigit(int c)
    • isJSSpace

      private static boolean isJSSpace(int c)
    • isJSFormatChar

      private static boolean isJSFormatChar(int c)
    • readRegExp

      void readRegExp(int startToken) throws IOException
      Parser calls the method when it gets / or /= in literal context.
      Throws:
      IOException
    • readAndClearRegExpFlags

      String readAndClearRegExpFlags()
    • getRawString

      String getRawString()
    • getTemplateLiteralChar

      private int getTemplateLiteralChar() throws IOException
      Throws:
      IOException
    • ungetTemplateLiteralChar

      private void ungetTemplateLiteralChar(int c)
    • matchTemplateLiteralChar

      private boolean matchTemplateLiteralChar(int test) throws IOException
      Throws:
      IOException
    • peekTemplateLiteralChar

      private int peekTemplateLiteralChar() throws IOException
      Throws:
      IOException
    • readTemplateLiteral

      int readTemplateLiteral(boolean isTaggedLiteral) throws IOException
      Throws:
      IOException
    • isXMLAttribute

      boolean isXMLAttribute()
    • getFirstXMLToken

      int getFirstXMLToken() throws IOException
      Throws:
      IOException
    • getNextXMLToken

      int getNextXMLToken() throws IOException
      Throws:
      IOException
    • readQuotedString

      private boolean readQuotedString(int quote) throws IOException
      Throws:
      IOException
    • readXmlComment

      private boolean readXmlComment() throws IOException
      Throws:
      IOException
    • readCDATA

      private boolean readCDATA() throws IOException
      Throws:
      IOException
    • readEntity

      private boolean readEntity() throws IOException
      Throws:
      IOException
    • readPI

      private boolean readPI() throws IOException
      Throws:
      IOException
    • getStringFromBuffer

      private String getStringFromBuffer()
    • addToString

      private void addToString(int c)
    • canUngetChar

      private boolean canUngetChar()
    • ungetChar

      private void ungetChar(int c)
    • matchChar

      private boolean matchChar(int test) throws IOException
      Throws:
      IOException
    • peekChar

      private int peekChar() throws IOException
      Throws:
      IOException
    • getChar

      private int getChar() throws IOException
      Throws:
      IOException
    • getChar

      private int getChar(boolean skipFormattingChars) throws IOException
      Throws:
      IOException
    • getChar

      private int getChar(boolean skipFormattingChars, boolean ignoreLineEnd) throws IOException
      Throws:
      IOException
    • getCharIgnoreLineEnd

      private int getCharIgnoreLineEnd() throws IOException
      Throws:
      IOException
    • getCharIgnoreLineEnd

      private int getCharIgnoreLineEnd(boolean skipFormattingChars) throws IOException
      Throws:
      IOException
    • ungetCharIgnoreLineEnd

      private void ungetCharIgnoreLineEnd(int c)
    • skipLine

      private void skipLine() throws IOException
      Throws:
      IOException
    • getOffset

      final int getOffset()
      Returns the offset into the current line.
    • charAt

      private final int charAt(int index)
    • substring

      private final String substring(int beginIndex, int endIndex)
    • getLine

      final String getLine()
    • getLine

      final String getLine(int position, int[] linep)
    • fillSourceBuffer

      private boolean fillSourceBuffer() throws IOException
      Throws:
      IOException
    • getCursor

      public int getCursor()
      Return the current position of the scanner cursor.
    • getTokenBeg

      public int getTokenBeg()
      Return the absolute source offset of the last scanned token.
    • getTokenEnd

      public int getTokenEnd()
      Return the absolute source end-offset of the last scanned token.
    • getTokenLength

      public int getTokenLength()
      Return tokenEnd - tokenBeg
    • getCommentType

      public Token.CommentType getCommentType()
      Return the type of the last scanned comment.
      Returns:
      type of last scanned comment, or 0 if none have been scanned.
    • markCommentStart

      private void markCommentStart()
    • markCommentStart

      private void markCommentStart(String prefix)
    • isMarkingComment

      private boolean isMarkingComment()
    • getAndResetCurrentComment

      final String getAndResetCurrentComment()
    • convertLastCharToHex

      private static String convertLastCharToHex(String str)