aspclassic-core/AspClassic.Parser/Scanner.cs

#define DEBUG
using System;
using System.Collections.Generic;
using System.Diagnostics;
using System.Globalization;
using System.IO;
using System.Runtime.CompilerServices;
using System.Text;
using Microsoft.VisualBasic;
using Microsoft.VisualBasic.CompilerServices;

namespace AspClassic.Parser;

/// <summary>
/// A lexical analyzer for Visual Basic .NET. It produces a stream of lexical tokens.
/// </summary>
public sealed class Scanner : IDisposable
{
	private TextReader _Source;

	private char _PeekCache;

	private bool _PeekCacheHasValue;

	private char _PeekAheadCache;

	private bool _PeekAheadCacheHasValue;

	private int _Index;

	private int _Line;

	private int _Column;

	private List<Token> _Tokens;

	private int _Position;

	private bool _Disposed;

	private int _TabSpaces;

	private LanguageVersion _Version;

	[SpecialName]
	private Dictionary<string, TypeCharacter> _0024STATIC_0024ScanPossibleTypeCharacter_00242011182E41182E4_0024TypeCharacterTable;

	/// <summary>
	/// How many columns a tab character should be considered.
	/// </summary>
	public int TabSpaces
	{
		get
		{
			return _TabSpaces;
		}
		set
		{
			if (value < 1)
			{
				throw new ArgumentException("Tabs cannot represent less than one space.");
			}
			_TabSpaces = value;
		}
	}

	/// <summary>
	/// The version of Visual Basic this scanner operates on.
	/// </summary>
	public LanguageVersion Version => _Version;

	private Location CurrentLocation => new Location(_Index, _Line, _Column);

	/// <summary>
	/// Whether the stream is positioned on the first token.
	/// </summary>
	public bool IsOnFirstToken => _Position == -1;

	/// <summary>
	/// Constructs a scanner for a string.
	/// </summary>
	/// <param name="source">The string to scan.</param>
	public Scanner(string source)
	{
		_PeekCacheHasValue = false;
		_PeekAheadCacheHasValue = false;
		_Index = 0;
		_Line = 1;
		_Column = 1;
		_Tokens = new List<Token>();
		_Position = -1;
		_Disposed = false;
		_TabSpaces = 4;
		_Version = LanguageVersion.VisualBasic80;
		if (source == null)
		{
			throw new ArgumentNullException("Source");
		}
		_Source = new StringReader(source);
	}

	/// <summary>
	/// Constructs a scanner for a string.
	/// </summary>
	/// <param name="source">The string to scan.</param>
	/// <param name="version">The language version to parse.</param>
	public Scanner(string source, LanguageVersion version)
	{
		_PeekCacheHasValue = false;
		_PeekAheadCacheHasValue = false;
		_Index = 0;
		_Line = 1;
		_Column = 1;
		_Tokens = new List<Token>();
		_Position = -1;
		_Disposed = false;
		_TabSpaces = 4;
		_Version = LanguageVersion.VisualBasic80;
		if (source == null)
		{
			throw new ArgumentNullException("Source");
		}
		if (version != LanguageVersion.VisualBasic71 && version != LanguageVersion.VisualBasic80)
		{
			throw new ArgumentOutOfRangeException("Version");
		}
		_Source = new StringReader(source);
		_Version = version;
	}

	/// <summary>
	/// Constructs a scanner for a stream.
	/// </summary>
	/// <param name="source">The stream to scan.</param>
	public Scanner(Stream source)
	{
		_PeekCacheHasValue = false;
		_PeekAheadCacheHasValue = false;
		_Index = 0;
		_Line = 1;
		_Column = 1;
		_Tokens = new List<Token>();
		_Position = -1;
		_Disposed = false;
		_TabSpaces = 4;
		_Version = LanguageVersion.VisualBasic80;
		if (source == null)
		{
			throw new ArgumentNullException("Source");
		}
		_Source = new StreamReader(source);
	}

	/// <summary>
	/// Constructs a scanner for a stream.
	/// </summary>
	/// <param name="source">The stream to scan.</param>
	/// <param name="version">The language version to parse.</param>
	public Scanner(Stream source, LanguageVersion version)
	{
		_PeekCacheHasValue = false;
		_PeekAheadCacheHasValue = false;
		_Index = 0;
		_Line = 1;
		_Column = 1;
		_Tokens = new List<Token>();
		_Position = -1;
		_Disposed = false;
		_TabSpaces = 4;
		_Version = LanguageVersion.VisualBasic80;
		if (source == null)
		{
			throw new ArgumentNullException("Source");
		}
		if (version != LanguageVersion.VisualBasic71 && version != LanguageVersion.VisualBasic80)
		{
			throw new ArgumentOutOfRangeException("Version");
		}
		_Source = new StreamReader(source);
		_Version = version;
	}

	/// <summary>
	/// Constructs a canner for a general TextReader.
	/// </summary>
	/// <param name="source">The TextReader to scan.</param>
	public Scanner(TextReader source)
	{
		_PeekCacheHasValue = false;
		_PeekAheadCacheHasValue = false;
		_Index = 0;
		_Line = 1;
		_Column = 1;
		_Tokens = new List<Token>();
		_Position = -1;
		_Disposed = false;
		_TabSpaces = 4;
		_Version = LanguageVersion.VisualBasic80;
		if (source == null)
		{
			throw new ArgumentNullException("Source");
		}
		_Source = source;
	}

	/// <summary>
	/// Constructs a canner for a general TextReader.
	/// </summary>
	/// <param name="source">The TextReader to scan.</param>
	/// <param name="version">The language version to parse.</param>
	public Scanner(TextReader source, LanguageVersion version)
	{
		_PeekCacheHasValue = false;
		_PeekAheadCacheHasValue = false;
		_Index = 0;
		_Line = 1;
		_Column = 1;
		_Tokens = new List<Token>();
		_Position = -1;
		_Disposed = false;
		_TabSpaces = 4;
		_Version = LanguageVersion.VisualBasic80;
		if (source == null)
		{
			throw new ArgumentNullException("Source");
		}
		if (version != LanguageVersion.VisualBasic71 && version != LanguageVersion.VisualBasic80)
		{
			throw new ArgumentOutOfRangeException("Version");
		}
		_Source = source;
		_Version = version;
	}

	/// <summary>
	/// Closes/disposes the scanner.
	/// </summary>
	public void Close()
	{
		if (!_Disposed)
		{
			_Disposed = true;
			_Source.Close();
		}
	}

	void IDisposable.Dispose()
	{
		//ILSpy generated this explicit interface implementation from .override directive in Close
		this.Close();
	}

	private char ReadChar()
	{
		char c;
		if (_PeekCacheHasValue)
		{
			c = _PeekCache;
			_PeekCacheHasValue = false;
			if (_PeekAheadCacheHasValue)
			{
				_PeekCache = _PeekAheadCache;
				_PeekCacheHasValue = true;
				_PeekAheadCacheHasValue = false;
			}
		}
		else
		{
			Debug.Assert(!_PeekAheadCacheHasValue, "Cache incorrect!");
			c = Strings.ChrW(_Source.Read());
		}
		checked
		{
			_Index++;
			if (c == '\t')
			{
				_Column += _TabSpaces;
			}
			else
			{
				_Column++;
			}
			return c;
		}
	}

	private char PeekChar()
	{
		if (!_PeekCacheHasValue)
		{
			_PeekCache = Strings.ChrW(_Source.Read());
			_PeekCacheHasValue = true;
		}
		return _PeekCache;
	}

	private char PeekAheadChar()
	{
		if (!_PeekAheadCacheHasValue)
		{
			if (!_PeekCacheHasValue)
			{
				PeekChar();
			}
			_PeekAheadCache = Strings.ChrW(_Source.Read());
			_PeekAheadCacheHasValue = true;
		}
		return _PeekAheadCache;
	}

	private Span SpanFrom(Location start)
	{
		return new Span(start, CurrentLocation);
	}

	private static bool IsAlphaClass(UnicodeCategory c)
	{
		return c == UnicodeCategory.UppercaseLetter || c == UnicodeCategory.LowercaseLetter || c == UnicodeCategory.TitlecaseLetter || c == UnicodeCategory.OtherLetter || c == UnicodeCategory.ModifierLetter || c == UnicodeCategory.LetterNumber;
	}

	private static bool IsNumericClass(UnicodeCategory c)
	{
		return c == UnicodeCategory.DecimalDigitNumber;
	}

	private static bool IsUnderscoreClass(UnicodeCategory c)
	{
		return c == UnicodeCategory.ConnectorPunctuation;
	}

	private static bool IsSingleQuote(char c)
	{
		return c == '\'' || c == '＇' || c == '‘' || c == '’';
	}

	private static bool IsDoubleQuote(char c)
	{
		return c == '"' || c == '＂' || c == '“' || c == '”';
	}

	private static bool IsDigit(char c)
	{
		return (c >= '0' && c <= '9') || (c >= '０' && c <= '９');
	}

	private static bool IsOctalDigit(char c)
	{
		return (c >= '0' && c <= '7') || (c >= '０' && c <= '７');
	}

	private static bool IsHexDigit(char c)
	{
		return IsDigit(c) || (c >= 'a' && c <= 'f') || (c >= 'A' && c <= 'F') || (c >= 'ａ' && c <= 'ｆ') || (c >= 'Ａ' && c <= 'Ｆ');
	}

	private static bool IsEquals(char c)
	{
		return c == '=' || c == '＝';
	}

	private static bool IsLessThan(char c)
	{
		return c == '<' || c == '＜';
	}

	private static bool IsGreaterThan(char c)
	{
		return c == '>' || c == '＞';
	}

	private static bool IsAmpersand(char c)
	{
		return c == '&' || c == '＆';
	}

	private static bool IsUnderscore(char c)
	{
		return IsUnderscoreClass(char.GetUnicodeCategory(c));
	}

	private static bool IsHexDesignator(char c)
	{
		return c == 'H' || c == 'h' || c == 'ｈ' || c == 'Ｈ';
	}

	private static bool IsOctalDesignator(char c)
	{
		return c == 'O' || c == 'o' || c == 'Ｏ' || c == 'ｏ';
	}

	private static bool IsPeriod(char c)
	{
		return c == '.' || c == '．';
	}

	private static bool IsExponentDesignator(char c)
	{
		return c == 'e' || c == 'E' || c == 'ｅ' || c == 'Ｅ';
	}

	private static bool IsPlus(char c)
	{
		return c == '+' || c == '＋';
	}

	private static bool IsMinus(char c)
	{
		return c == '-' || c == '－';
	}

	private static bool IsForwardSlash(char c)
	{
		return c == '/' || c == '／';
	}

	private static bool IsColon(char c)
	{
		return c == ':' || c == '：';
	}

	private static bool IsPound(char c)
	{
		return c == '#' || c == '＃';
	}

	private static bool IsA(char c)
	{
		return c == 'a' || c == 'ａ' || c == 'A' || c == 'Ａ';
	}

	private static bool IsP(char c)
	{
		return c == 'p' || c == 'ｐ' || c == 'P' || c == 'Ｐ';
	}

	private static bool IsM(char c)
	{
		return c == 'm' || c == 'ｍ' || c == 'M' || c == 'Ｍ';
	}

	private static bool IsCharDesignator(char c)
	{
		return c == 'c' || c == 'C' || c == 'ｃ' || c == 'Ｃ';
	}

	private static bool IsLeftBracket(char c)
	{
		return c == '[' || c == '［';
	}

	private static bool IsRightBracket(char c)
	{
		return c == ']' || c == '］';
	}

	private static bool IsUnsignedTypeChar(char c)
	{
		return c == 'u' || c == 'U' || c == 'Ｕ' || c == 'ｕ';
	}

	private static bool IsIdentifier(char c)
	{
		UnicodeCategory CharClass = char.GetUnicodeCategory(c);
		return IsAlphaClass(CharClass) || IsNumericClass(CharClass) || CharClass == UnicodeCategory.SpacingCombiningMark || CharClass == UnicodeCategory.NonSpacingMark || CharClass == UnicodeCategory.Format || IsUnderscoreClass(CharClass);
	}

	internal static char MakeHalfWidth(char c)
	{
		if (c < '！' || c > '～')
		{
			return c;
		}
		checked
		{
			return Strings.ChrW(unchecked((int)c) - 65280 + 32);
		}
	}

	internal static char MakeFullWidth(char c)
	{
		if (c < '!' || c > '~')
		{
			return c;
		}
		checked
		{
			return Strings.ChrW(unchecked((int)c) + 65280 - 32);
		}
	}

	internal static string MakeFullWidth(string s)
	{
		StringBuilder Builder = new StringBuilder(s);
		checked
		{
			int num = Builder.Length - 1;
			for (int Index = 0; Index <= num; Index++)
			{
				Builder[Index] = MakeFullWidth(Builder[Index]);
			}
			return Builder.ToString();
		}
	}

	private TypeCharacter ScanPossibleTypeCharacter(TypeCharacter ValidTypeCharacters)
	{
		char TypeChar = PeekChar();
		if (_0024STATIC_0024ScanPossibleTypeCharacter_00242011182E41182E4_0024TypeCharacterTable == null)
		{
			Dictionary<string, TypeCharacter> Table = new Dictionary<string, TypeCharacter>(StringComparer.InvariantCultureIgnoreCase);
			string[] TypeCharacters = new string[15]
			{
				"$", "%", "&", "S", "I", "L", "!", "#", "@", "F",
				"R", "D", "US", "UI", "UL"
			};
			TypeCharacter TypeCharacter2 = TypeCharacter.StringSymbol;
			int num = checked(TypeCharacters.Length - 1);
			for (int Index = 0; Index <= num; Index = checked(Index + 1))
			{
				Dictionary<string, TypeCharacter> dictionary = Table;
				dictionary.Add(TypeCharacters[Index], TypeCharacter2);
				dictionary.Add(MakeFullWidth(TypeCharacters[Index]), TypeCharacter2);
				dictionary = null;
				TypeCharacter2 = (TypeCharacter)((int)TypeCharacter2 << 1);
			}
			_0024STATIC_0024ScanPossibleTypeCharacter_00242011182E41182E4_0024TypeCharacterTable = Table;
		}
		string TypeString = ((!IsUnsignedTypeChar(TypeChar) || _Version <= LanguageVersion.VisualBasic71) ? Conversions.ToString(TypeChar) : (Conversions.ToString(TypeChar) + Conversions.ToString(PeekAheadChar())));
		if (_0024STATIC_0024ScanPossibleTypeCharacter_00242011182E41182E4_0024TypeCharacterTable.ContainsKey(TypeString))
		{
			TypeCharacter TypeCharacter = _0024STATIC_0024ScanPossibleTypeCharacter_00242011182E41182E4_0024TypeCharacterTable[TypeString];
			if ((TypeCharacter & ValidTypeCharacters) != 0)
			{
				if (TypeCharacter == TypeCharacter.SingleSymbol && CanStartIdentifier(PeekAheadChar()))
				{
					return TypeCharacter.None;
				}
				ReadChar();
				if (IsUnsignedTypeChar(TypeChar))
				{
					ReadChar();
				}
				return TypeCharacter;
			}
		}
		return TypeCharacter.None;
	}

	private PunctuatorToken ScanPossibleMultiCharacterPunctuator(char leadingCharacter, Location start)
	{
		char NextChar = PeekChar();
		string PunctuatorString = Conversions.ToString(leadingCharacter);
		Debug.Assert(PunctuatorToken.TokenTypeFromString(Conversions.ToString(leadingCharacter)) != TokenType.None);
		TokenType Punctuator;
		if (IsEquals(NextChar) || IsLessThan(NextChar) || IsGreaterThan(NextChar))
		{
			PunctuatorString += Conversions.ToString(NextChar);
			Punctuator = PunctuatorToken.TokenTypeFromString(PunctuatorString);
			if (Punctuator != 0)
			{
				ReadChar();
				if ((Punctuator == TokenType.LessThanLessThan || Punctuator == TokenType.GreaterThanGreaterThan) && IsEquals(PeekChar()))
				{
					PunctuatorString += Conversions.ToString(ReadChar());
					Punctuator = PunctuatorToken.TokenTypeFromString(PunctuatorString);
				}
				return new PunctuatorToken(Punctuator, SpanFrom(start));
			}
		}
		Punctuator = PunctuatorToken.TokenTypeFromString(Conversions.ToString(leadingCharacter));
		return new PunctuatorToken(Punctuator, SpanFrom(start));
	}

	private Token ScanNumericLiteral()
	{
		Location Start = CurrentLocation;
		StringBuilder Literal = new StringBuilder();
		IntegerBase Base = IntegerBase.Decimal;
		TypeCharacter TypeCharacter = TypeCharacter.None;
		Debug.Assert(CanStartNumericLiteral());
		checked
		{
			if (IsAmpersand(PeekChar()))
			{
				Literal.Append(MakeHalfWidth(ReadChar()));
				if (IsHexDesignator(PeekChar()))
				{
					Literal.Append(MakeHalfWidth(ReadChar()));
					Base = IntegerBase.Hexadecimal;
					while (IsHexDigit(PeekChar()))
					{
						Literal.Append(MakeHalfWidth(ReadChar()));
					}
				}
				else if (IsOctalDesignator(PeekChar()))
				{
					Literal.Append(MakeHalfWidth(ReadChar()));
					Base = IntegerBase.Octal;
					while (IsOctalDigit(PeekChar()))
					{
						Literal.Append(MakeHalfWidth(ReadChar()));
					}
				}
				else
				{
					if (!IsOctalDigit(PeekChar()))
					{
						return ScanPossibleMultiCharacterPunctuator('&', Start);
					}
					Base = IntegerBase.Octal;
					Literal.Append('O');
					while (IsOctalDigit(PeekChar()))
					{
						Literal.Append(MakeHalfWidth(ReadChar()));
					}
					if (IsAmpersand(PeekChar()))
					{
						ReadChar();
					}
				}
				if (Literal.Length > 2)
				{
					TypeCharacter = ScanPossibleTypeCharacter(TypeCharacter.IntegerSymbol | TypeCharacter.LongSymbol | TypeCharacter.ShortChar | TypeCharacter.IntegerChar | TypeCharacter.LongChar | TypeCharacter.UnsignedShortChar | TypeCharacter.UnsignedIntegerChar | TypeCharacter.UnsignedLongChar);
					try
					{
						switch (TypeCharacter)
						{
						case TypeCharacter.ShortChar:
						{
							long Value3 = Conversions.ToLong(Literal.ToString());
							if (Value3 <= 65535)
							{
								if (Value3 > 32767)
								{
									Value3 = -(65536 - Value3);
								}
								if (Value3 >= -32768 && Value3 <= 32767)
								{
									return new IntegerLiteralToken((short)Value3, Base, TypeCharacter, SpanFrom(Start));
								}
							}
							break;
						}
						case TypeCharacter.UnsignedShortChar:
						{
							ulong Value4 = Conversions.ToULong(Literal.ToString());
							if (decimal.Compare(new decimal(Value4), new decimal(65535L)) <= 0 && Value4 >= 0 && Value4 <= 65535)
							{
								return new UnsignedIntegerLiteralToken((ushort)Value4, Base, TypeCharacter, SpanFrom(Start));
							}
							break;
						}
						case TypeCharacter.IntegerSymbol:
						case TypeCharacter.IntegerChar:
						{
							long Value = Conversions.ToLong(Literal.ToString());
							if (Value <= uint.MaxValue)
							{
								if (Value > int.MaxValue)
								{
									Value = -(4294967296L - Value);
								}
								if (Value >= int.MinValue && Value <= int.MaxValue)
								{
									return new IntegerLiteralToken((int)Value, Base, TypeCharacter, SpanFrom(Start));
								}
							}
							break;
						}
						case TypeCharacter.UnsignedIntegerChar:
						{
							ulong Value2 = Conversions.ToULong(Literal.ToString());
							if (decimal.Compare(new decimal(Value2), new decimal(4294967295L)) <= 0 && Value2 >= 0 && Value2 <= uint.MaxValue)
							{
								return new UnsignedIntegerLiteralToken((uint)Value2, Base, TypeCharacter, SpanFrom(Start));
							}
							break;
						}
						case TypeCharacter.LongSymbol:
						case TypeCharacter.LongChar:
							return new IntegerLiteralToken(ParseInt(Literal.ToString()), Base, TypeCharacter, SpanFrom(Start));
						case TypeCharacter.UnsignedLongChar:
							return new UnsignedIntegerLiteralToken(Conversions.ToULong(Literal.ToString()), Base, TypeCharacter, SpanFrom(Start));
						default:
							TypeCharacter = TypeCharacter.None;
							return new IntegerLiteralToken(ParseInt(Literal.ToString()), Base, TypeCharacter, SpanFrom(Start));
						}
					}
					catch (OverflowException ex7)
					{
						ProjectData.SetProjectError(ex7);
						OverflowException ex6 = ex7;
						Token ScanNumericLiteral = new ErrorToken(SyntaxErrorType.InvalidIntegerLiteral, SpanFrom(Start));
						ProjectData.ClearProjectError();
						return ScanNumericLiteral;
					}
					catch (InvalidCastException ex8)
					{
						ProjectData.SetProjectError(ex8);
						InvalidCastException ex5 = ex8;
						Token ScanNumericLiteral = new ErrorToken(SyntaxErrorType.InvalidIntegerLiteral, SpanFrom(Start));
						ProjectData.ClearProjectError();
						return ScanNumericLiteral;
					}
				}
				return new ErrorToken(SyntaxErrorType.InvalidIntegerLiteral, SpanFrom(Start));
			}
			while (IsDigit(PeekChar()))
			{
				Literal.Append(MakeHalfWidth(ReadChar()));
			}
			if (IsPeriod(PeekChar()) || IsExponentDesignator(PeekChar()))
			{
				SyntaxErrorType ErrorType = SyntaxErrorType.None;
				if (IsPeriod(PeekChar()))
				{
					Literal.Append(MakeHalfWidth(ReadChar()));
					if (!IsDigit(PeekChar()) & (Literal.Length == 1))
					{
						return new PunctuatorToken(TokenType.Period, SpanFrom(Start));
					}
					while (IsDigit(PeekChar()))
					{
						Literal.Append(MakeHalfWidth(ReadChar()));
					}
				}
				if (IsExponentDesignator(PeekChar()))
				{
					Literal.Append(MakeHalfWidth(ReadChar()));
					if (IsPlus(PeekChar()) || IsMinus(PeekChar()))
					{
						Literal.Append(MakeHalfWidth(ReadChar()));
					}
					if (!IsDigit(PeekChar()))
					{
						return new ErrorToken(SyntaxErrorType.InvalidFloatingPointLiteral, SpanFrom(Start));
					}
					while (IsDigit(PeekChar()))
					{
						Literal.Append(MakeHalfWidth(ReadChar()));
					}
				}
				TypeCharacter = ScanPossibleTypeCharacter(TypeCharacter.SingleSymbol | TypeCharacter.DoubleSymbol | TypeCharacter.DecimalSymbol | TypeCharacter.SingleChar | TypeCharacter.DoubleChar | TypeCharacter.DecimalChar);
				try
				{
					switch (TypeCharacter)
					{
					case TypeCharacter.DecimalSymbol:
					case TypeCharacter.DecimalChar:
						ErrorType = SyntaxErrorType.InvalidDecimalLiteral;
						return new DecimalLiteralToken(Conversions.ToDecimal(Literal.ToString()), TypeCharacter, SpanFrom(Start));
					case TypeCharacter.SingleSymbol:
					case TypeCharacter.SingleChar:
						ErrorType = SyntaxErrorType.InvalidFloatingPointLiteral;
						return new FloatingPointLiteralToken(Conversions.ToSingle(Literal.ToString()), TypeCharacter, SpanFrom(Start));
					case TypeCharacter.DoubleSymbol:
					case TypeCharacter.DoubleChar:
						ErrorType = SyntaxErrorType.InvalidFloatingPointLiteral;
						return new FloatingPointLiteralToken(Conversions.ToDouble(Literal.ToString()), TypeCharacter, SpanFrom(Start));
					default:
						ErrorType = SyntaxErrorType.InvalidFloatingPointLiteral;
						TypeCharacter = TypeCharacter.None;
						return new FloatingPointLiteralToken(Conversions.ToDouble(Literal.ToString()), TypeCharacter, SpanFrom(Start));
					}
				}
				catch (OverflowException ex9)
				{
					ProjectData.SetProjectError(ex9);
					OverflowException ex2 = ex9;
					Token ScanNumericLiteral = new ErrorToken(ErrorType, SpanFrom(Start));
					ProjectData.ClearProjectError();
					return ScanNumericLiteral;
				}
				catch (InvalidCastException ex10)
				{
					ProjectData.SetProjectError(ex10);
					InvalidCastException ex = ex10;
					Token ScanNumericLiteral = new ErrorToken(ErrorType, SpanFrom(Start));
					ProjectData.ClearProjectError();
					return ScanNumericLiteral;
				}
			}
			SyntaxErrorType ErrorType2 = SyntaxErrorType.None;
			TypeCharacter = ScanPossibleTypeCharacter(TypeCharacter.IntegerSymbol | TypeCharacter.LongSymbol | TypeCharacter.ShortChar | TypeCharacter.IntegerChar | TypeCharacter.LongChar | TypeCharacter.SingleSymbol | TypeCharacter.DoubleSymbol | TypeCharacter.DecimalSymbol | TypeCharacter.SingleChar | TypeCharacter.DoubleChar | TypeCharacter.DecimalChar | TypeCharacter.UnsignedShortChar | TypeCharacter.UnsignedIntegerChar | TypeCharacter.UnsignedLongChar);
			try
			{
				switch (TypeCharacter)
				{
				case TypeCharacter.ShortChar:
					ErrorType2 = SyntaxErrorType.InvalidIntegerLiteral;
					return new IntegerLiteralToken(Conversions.ToShort(Literal.ToString()), Base, TypeCharacter, SpanFrom(Start));
				case TypeCharacter.UnsignedShortChar:
					ErrorType2 = SyntaxErrorType.InvalidIntegerLiteral;
					return new UnsignedIntegerLiteralToken(Conversions.ToUShort(Literal.ToString()), Base, TypeCharacter, SpanFrom(Start));
				case TypeCharacter.IntegerSymbol:
				case TypeCharacter.IntegerChar:
					ErrorType2 = SyntaxErrorType.InvalidIntegerLiteral;
					return new IntegerLiteralToken(Conversions.ToInteger(Literal.ToString()), Base, TypeCharacter, SpanFrom(Start));
				case TypeCharacter.UnsignedIntegerChar:
					ErrorType2 = SyntaxErrorType.InvalidIntegerLiteral;
					return new UnsignedIntegerLiteralToken(Conversions.ToUInteger(Literal.ToString()), Base, TypeCharacter, SpanFrom(Start));
				case TypeCharacter.LongSymbol:
				case TypeCharacter.LongChar:
					ErrorType2 = SyntaxErrorType.InvalidIntegerLiteral;
					return new IntegerLiteralToken(Conversions.ToInteger(Literal.ToString()), Base, TypeCharacter, SpanFrom(Start));
				case TypeCharacter.UnsignedLongChar:
					ErrorType2 = SyntaxErrorType.InvalidIntegerLiteral;
					return new UnsignedIntegerLiteralToken(Conversions.ToULong(Literal.ToString()), Base, TypeCharacter, SpanFrom(Start));
				case TypeCharacter.DecimalSymbol:
				case TypeCharacter.DecimalChar:
					ErrorType2 = SyntaxErrorType.InvalidDecimalLiteral;
					return new DecimalLiteralToken(Conversions.ToDecimal(Literal.ToString()), TypeCharacter, SpanFrom(Start));
				case TypeCharacter.SingleSymbol:
				case TypeCharacter.SingleChar:
					ErrorType2 = SyntaxErrorType.InvalidFloatingPointLiteral;
					return new FloatingPointLiteralToken(Conversions.ToSingle(Literal.ToString()), TypeCharacter, SpanFrom(Start));
				case TypeCharacter.DoubleSymbol:
				case TypeCharacter.DoubleChar:
					ErrorType2 = SyntaxErrorType.InvalidFloatingPointLiteral;
					return new FloatingPointLiteralToken(Conversions.ToDouble(Literal.ToString()), TypeCharacter, SpanFrom(Start));
				default:
					ErrorType2 = SyntaxErrorType.InvalidIntegerLiteral;
					return new IntegerLiteralToken(Conversions.ToInteger(Literal.ToString()), Base, TypeCharacter.None, SpanFrom(Start));
				}
			}
			catch (OverflowException ex11)
			{
				ProjectData.SetProjectError(ex11);
				OverflowException ex4 = ex11;
				Token ScanNumericLiteral = new ErrorToken(ErrorType2, SpanFrom(Start));
				ProjectData.ClearProjectError();
				return ScanNumericLiteral;
			}
			catch (InvalidCastException ex12)
			{
				ProjectData.SetProjectError(ex12);
				InvalidCastException ex3 = ex12;
				Token ScanNumericLiteral = new ErrorToken(ErrorType2, SpanFrom(Start));
				ProjectData.ClearProjectError();
				return ScanNumericLiteral;
			}
		}
	}

	private bool CanStartNumericLiteral()
	{
		return IsPeriod(PeekChar()) || IsAmpersand(PeekChar()) || IsDigit(PeekChar());
	}

	private long ReadIntegerLiteral()
	{
		long Value = 0L;
		checked
		{
			while (IsDigit(PeekChar()))
			{
				char c = MakeHalfWidth(ReadChar());
				Value *= 10;
				Value += unchecked((int)c) - 48;
			}
			return Value;
		}
	}

	private Token ScanDateLiteral()
	{
		Location Start = CurrentLocation;
		int Month = 0;
		int Day = 0;
		int Year = 0;
		int Hour = 0;
		int Minute = 0;
		int Second = 0;
		bool HaveDateValue = false;
		bool HaveTimeValue = false;
		Debug.Assert(CanStartDateLiteral());
		ReadChar();
		Location PossibleEnd = CurrentLocation;
		EatWhitespace();
		if (!IsDigit(PeekChar()))
		{
			return new PunctuatorToken(TokenType.Pound, new Span(Start, PossibleEnd));
		}
		long Value = ReadIntegerLiteral();
		EatWhitespace();
		if (!IsForwardSlash(PeekChar()) && !IsMinus(PeekChar()))
		{
			goto IL_024d;
		}
		char Separator = ReadChar();
		HaveDateValue = true;
		checked
		{
			if (Value >= 1 && Value <= 12)
			{
				Month = (int)Value;
				EatWhitespace();
				if (IsDigit(PeekChar()))
				{
					Value = ReadIntegerLiteral();
					if (Value >= 1 && Value <= 31)
					{
						Day = (int)Value;
						EatWhitespace();
						if (PeekChar() == Separator)
						{
							ReadChar();
							EatWhitespace();
							if (IsDigit(PeekChar()))
							{
								Location YearStart = CurrentLocation;
								Value = ReadIntegerLiteral();
								if (Value >= 1 && Value <= 9999)
								{
									Year = (int)Value;
									if (CurrentLocation.Column - YearStart.Column == 2)
									{
										Year = ((Year <= 30) ? (Year + 1900) : (Year + 2000));
									}
									else if (CurrentLocation.Column - YearStart.Column != 4)
									{
										goto IL_042a;
									}
									if (Day <= DateTime.DaysInMonth(Year, Month))
									{
										EatWhitespace();
										if (IsDigit(PeekChar()))
										{
											Value = ReadIntegerLiteral();
											if (!IsColon(PeekChar()))
											{
												goto IL_042a;
											}
										}
										goto IL_024d;
									}
								}
							}
						}
					}
				}
			}
			goto IL_042a;
		}
		IL_03ed:
		if (IsPound(PeekChar()))
		{
			ReadChar();
			if (HaveTimeValue || HaveDateValue)
			{
				if (HaveDateValue)
				{
					if (HaveTimeValue)
					{
						return new DateLiteralToken(new DateTime(Year, Month, Day, Hour, Minute, Second), SpanFrom(Start));
					}
					return new DateLiteralToken(new DateTime(Year, Month, Day), SpanFrom(Start));
				}
				return new DateLiteralToken(new DateTime(1, 1, 1, Hour, Minute, Second), SpanFrom(Start));
			}
		}
		goto IL_042a;
		IL_0327:
		EatWhitespace();
		if (IsA(PeekChar()))
		{
			ReadChar();
			if (IsM(PeekChar()))
			{
				ReadChar();
				if (Hour >= 1 && Hour <= 12)
				{
					goto IL_03ed;
				}
			}
		}
		else
		{
			if (!IsP(PeekChar()))
			{
				goto IL_03ed;
			}
			ReadChar();
			if (IsM(PeekChar()))
			{
				ReadChar();
				if (Hour >= 1 && Hour <= 12)
				{
					Hour = checked(Hour + 12);
					if (Hour == 24)
					{
						Hour = 12;
					}
					goto IL_03ed;
				}
			}
		}
		goto IL_042a;
		IL_042a:
		while (!IsPound(PeekChar()) && !CanStartLineTerminator())
		{
			ReadChar();
		}
		if (IsPound(PeekChar()))
		{
			ReadChar();
		}
		return new ErrorToken(SyntaxErrorType.InvalidDateLiteral, SpanFrom(Start));
		IL_024d:
		if (!IsColon(PeekChar()))
		{
			goto IL_03ed;
		}
		ReadChar();
		HaveTimeValue = true;
		checked
		{
			if (Value >= 0 && Value <= 23)
			{
				Hour = (int)Value;
				if (IsDigit(PeekChar()))
				{
					Value = ReadIntegerLiteral();
					if (Value >= 0 && Value <= 59)
					{
						Minute = (int)Value;
						if (!IsColon(PeekChar()))
						{
							goto IL_0327;
						}
						ReadChar();
						if (IsDigit(PeekChar()))
						{
							Value = ReadIntegerLiteral();
							if (Value >= 0 && Value <= 59)
							{
								Second = (int)Value;
								goto IL_0327;
							}
						}
					}
				}
			}
			goto IL_042a;
		}
	}

	private bool CanStartDateLiteral()
	{
		return IsPound(PeekChar());
	}

	private Token ScanStringLiteral()
	{
		Location Start = CurrentLocation;
		StringBuilder s = new StringBuilder();
		Debug.Assert(CanStartStringLiteral());
		ReadChar();
		while (true)
		{
			if (!IsDoubleQuote(PeekChar()) && !CanStartLineTerminator())
			{
				s.Append(ReadChar());
				continue;
			}
			if (IsDoubleQuote(PeekChar()))
			{
				ReadChar();
				if (IsDoubleQuote(PeekChar()))
				{
					ReadChar();
					s.Append('"');
					continue;
				}
				break;
			}
			return new ErrorToken(SyntaxErrorType.InvalidStringLiteral, SpanFrom(Start));
		}
		if (IsCharDesignator(PeekChar()))
		{
			ReadChar();
			if (s.Length != 1)
			{
				return new ErrorToken(SyntaxErrorType.InvalidCharacterLiteral, SpanFrom(Start));
			}
			return new CharacterLiteralToken(s[0], SpanFrom(Start));
		}
		return new StringLiteralToken(s.ToString(), SpanFrom(Start));
	}

	private bool CanStartStringLiteral()
	{
		return IsDoubleQuote(PeekChar());
	}

	private Token ScanIdentifier()
	{
		Location Start = CurrentLocation;
		bool Escaped = false;
		TypeCharacter TypeCharacter = TypeCharacter.None;
		StringBuilder s = new StringBuilder();
		TokenType Type = TokenType.Identifier;
		TokenType UnreservedType = TokenType.Identifier;
		Debug.Assert(CanStartIdentifier());
		if (IsLeftBracket(PeekChar()))
		{
			Escaped = true;
			ReadChar();
			if (!CanStartNonEscapedIdentifier())
			{
				while (!IsRightBracket(PeekChar()) && !CanStartLineTerminator())
				{
					ReadChar();
				}
				if (IsRightBracket(PeekChar()))
				{
					ReadChar();
				}
				return new ErrorToken(SyntaxErrorType.InvalidEscapedIdentifier, SpanFrom(Start));
			}
		}
		s.Append(ReadChar());
		if (IsUnderscore(s[0]) && !IsIdentifier(PeekChar()))
		{
			Location End = CurrentLocation;
			EatWhitespace();
			if (CanStartLineTerminator())
			{
				ScanLineTerminator(produceToken: false);
				return null;
			}
			return new ErrorToken(SyntaxErrorType.InvalidIdentifier, new Span(Start, End));
		}
		while (IsIdentifier(PeekChar()))
		{
			s.Append(ReadChar());
		}
		string Identifier = s.ToString();
		if (Escaped)
		{
			if (!IsRightBracket(PeekChar()))
			{
				while (!IsRightBracket(PeekChar()) && !CanStartLineTerminator())
				{
					ReadChar();
				}
				if (IsRightBracket(PeekChar()))
				{
					ReadChar();
				}
				return new ErrorToken(SyntaxErrorType.InvalidEscapedIdentifier, SpanFrom(Start));
			}
			ReadChar();
		}
		else
		{
			Type = IdentifierToken.TokenTypeFromString(Identifier, _Version, IncludeUnreserved: false);
			if (Type == TokenType.REM)
			{
				return ScanComment(Start);
			}
			UnreservedType = IdentifierToken.TokenTypeFromString(Identifier, _Version, IncludeUnreserved: true);
			if (Type != TokenType.Identifier && TypeCharacter != 0)
			{
				if (_Version <= LanguageVersion.VisualBasic71)
				{
					return new ErrorToken(SyntaxErrorType.InvalidTypeCharacterOnKeyword, SpanFrom(Start));
				}
				Type = TokenType.Identifier;
			}
		}
		return new IdentifierToken(Type, UnreservedType, Identifier, Escaped, TypeCharacter, SpanFrom(Start));
	}

	private bool CanStartNonEscapedIdentifier()
	{
		return CanStartNonEscapedIdentifier(PeekChar());
	}

	private static bool CanStartNonEscapedIdentifier(char c)
	{
		UnicodeCategory CharClass = char.GetUnicodeCategory(c);
		return IsAlphaClass(CharClass) || IsUnderscoreClass(CharClass);
	}

	private bool CanStartIdentifier()
	{
		return CanStartIdentifier(PeekChar());
	}

	private static bool CanStartIdentifier(char c)
	{
		return IsLeftBracket(c) || CanStartNonEscapedIdentifier(c);
	}

	private CommentToken ScanComment()
	{
		StringBuilder s = new StringBuilder();
		Location Start = CurrentLocation;
		Debug.Assert(CanStartComment());
		ReadChar();
		while (!CanStartLineTerminator())
		{
			s.Append(ReadChar());
		}
		return new CommentToken(s.ToString(), isREM: false, SpanFrom(Start));
	}

	private CommentToken ScanComment(Location start)
	{
		StringBuilder s = new StringBuilder();
		while (!CanStartLineTerminator())
		{
			s.Append(ReadChar());
		}
		return new CommentToken(s.ToString(), isREM: true, SpanFrom(start));
	}

	private bool CanStartComment()
	{
		return IsSingleQuote(PeekChar());
	}

	private Token ScanLineTerminator(bool produceToken = true)
	{
		Location Start = CurrentLocation;
		Token Token = null;
		Debug.Assert(CanStartLineTerminator());
		checked
		{
			if (PeekChar() == '\uffff')
			{
				Token = new EndOfStreamToken(SpanFrom(Start));
			}
			else
			{
				if (ReadChar() == '\r' && PeekChar() == '\n')
				{
					ReadChar();
				}
				if (produceToken)
				{
					Token = new LineTerminatorToken(SpanFrom(Start));
				}
				_Line++;
				_Column = 1;
			}
			return Token;
		}
	}

	private bool CanStartLineTerminator()
	{
		char c = PeekChar();
		return c == '\r' || c == '\n' || c == '\u2028' || c == '\u2029' || c == '\uffff';
	}

	private bool EatWhitespace()
	{
		char c = PeekChar();
		bool EatWhitespace = default(bool);
		while (c == '\t' || char.GetUnicodeCategory(c) == UnicodeCategory.SpaceSeparator)
		{
			ReadChar();
			EatWhitespace = true;
			c = PeekChar();
		}
		return EatWhitespace;
	}

	private Token Read(bool advance)
	{
		Token CurrentToken = ((_Position <= -1) ? null : _Tokens[_Position]);
		if (CurrentToken != null && CurrentToken.Type == TokenType.EndOfStream)
		{
			return CurrentToken;
		}
		checked
		{
			if (_Position == _Tokens.Count - 1)
			{
				Token TokenRead;
				while (true)
				{
					EatWhitespace();
					if (CanStartLineTerminator())
					{
						TokenRead = ScanLineTerminator();
						break;
					}
					if (CanStartComment())
					{
						TokenRead = ScanComment();
						break;
					}
					if (CanStartIdentifier())
					{
						Token Token = ScanIdentifier();
						if (Token == null)
						{
							continue;
						}
						TokenRead = Token;
						break;
					}
					if (CanStartStringLiteral())
					{
						TokenRead = ScanStringLiteral();
						break;
					}
					if (CanStartDateLiteral())
					{
						TokenRead = ScanDateLiteral();
						break;
					}
					if (CanStartNumericLiteral())
					{
						TokenRead = ScanNumericLiteral();
						break;
					}
					Location Start = CurrentLocation;
					if (PunctuatorToken.TokenTypeFromString(Conversions.ToString(PeekChar())) != 0)
					{
						TokenRead = ScanPossibleMultiCharacterPunctuator(ReadChar(), Start);
						break;
					}
					ReadChar();
					TokenRead = new ErrorToken(SyntaxErrorType.InvalidCharacter, SpanFrom(Start));
					break;
				}
				_Tokens.Add(TokenRead);
			}
			if (advance)
			{
				_Position++;
				return _Tokens[_Position];
			}
			return _Tokens[_Position + 1];
		}
	}

	/// <summary>
	/// Seeks backwards in the stream position to a particular token.
	/// </summary>
	/// <param name="token">The token to seek back to.</param>
	/// <exception cref="T:System.ObjectDisposedException">Thrown when the scanner has been closed.</exception>
	/// <exception cref="T:System.ArgumentException">Thrown when token was not produced by this scanner.</exception>
	public void Seek(Token token)
	{
		int StartPosition = _Position;
		bool TokenFound = false;
		if (_Disposed)
		{
			throw new ObjectDisposedException("Scanner");
		}
		checked
		{
			if (StartPosition == _Tokens.Count - 1)
			{
				StartPosition--;
			}
			int num = StartPosition;
			int CurrentPosition;
			for (CurrentPosition = num; CurrentPosition >= -1; CurrentPosition += -1)
			{
				if (_Tokens[CurrentPosition + 1] == token)
				{
					TokenFound = true;
					break;
				}
			}
			if (!TokenFound)
			{
				throw new ArgumentException("Token not created by this scanner.");
			}
			_Position = CurrentPosition;
		}
	}

	/// <summary>
	/// Fetches the previous token in the stream.
	/// </summary>
	/// <returns>The previous token.</returns>
	/// <exception cref="T:System.ObjectDisposedException">Thrown when the scanner has been closed.</exception>
	/// <exception cref="T:System.InvalidOperationException">Thrown when the scanner is positioned on the first token.</exception>
	public Token Previous()
	{
		if (_Disposed)
		{
			throw new ObjectDisposedException("Scanner");
		}
		if (_Position == -1)
		{
			throw new InvalidOperationException("Scanner is positioned on the first token.");
		}
		return _Tokens[_Position];
	}

	/// <summary>
	/// Fetches the current token without advancing the stream position.
	/// </summary>
	/// <returns>The current token.</returns>
	/// <exception cref="T:System.ObjectDisposedException">Thrown when the scanner has been closed.</exception>
	public Token Peek()
	{
		if (_Disposed)
		{
			throw new ObjectDisposedException("Scanner");
		}
		return Read(advance: false);
	}

	/// <summary>
	/// Fetches the current token and advances the stream position.
	/// </summary>
	/// <returns>The current token.</returns>
	/// <exception cref="T:System.ObjectDisposedException">Thrown when the scanner has been closed.</exception>
	public Token Read()
	{
		if (_Disposed)
		{
			throw new ObjectDisposedException("Scanner");
		}
		return Read(advance: true);
	}

	/// <summary>
	/// Fetches more than one token at a time from the stream.
	/// </summary>
	/// <param name="buffer">The array to put the tokens into.</param>
	/// <param name="index">The location in the array to start putting the tokens into.</param>
	/// <param name="count">The number of tokens to read.</param>
	/// <returns>The number of tokens read.</returns>
	/// <exception cref="T:System.ObjectDisposedException">Thrown when the scanner has been closed.</exception>
	/// <exception cref="T:System.NullReferenceException">Thrown when the buffer is Nothing.</exception>
	/// <exception cref="T:System.ArgumentException">Thrown when the index or count is invalid, or when there isn't enough room in the buffer.</exception>
	public int ReadBlock(Token[] buffer, int index, int count)
	{
		int FinalCount = 0;
		if (buffer == null)
		{
			throw new ArgumentNullException("buffer");
		}
		if (index < 0 || count < 0)
		{
			throw new ArgumentException("Index or count cannot be negative.");
		}
		checked
		{
			if (buffer.Length - index < count)
			{
				throw new ArgumentException("Not enough room in buffer.");
			}
			if (_Disposed)
			{
				throw new ObjectDisposedException("Scanner");
			}
			while (count > 0)
			{
				Token CurrentToken = Read();
				if (CurrentToken.Type == TokenType.EndOfStream)
				{
					return FinalCount;
				}
				buffer[FinalCount + index] = CurrentToken;
				count--;
				FinalCount++;
			}
			return FinalCount;
		}
	}

	/// <summary>
	/// Reads all of the tokens between the current position and the end of the line (or the end of the stream).
	/// </summary>
	/// <returns>The tokens read.</returns>
	/// <exception cref="T:System.ObjectDisposedException">Thrown when the scanner has been closed.</exception>
	public Token[] ReadLine()
	{
		List<Token> TokenArray = new List<Token>();
		if (_Disposed)
		{
			throw new ObjectDisposedException("Scanner");
		}
		while ((Peek().Type != TokenType.EndOfStream) & (Peek().Type != TokenType.LineTerminator))
		{
			TokenArray.Add(Read());
		}
		return TokenArray.ToArray();
	}

	/// <summary>
	/// Reads all the tokens between the current position and the end of the stream.
	/// </summary>
	/// <returns>The tokens read.</returns>
	/// <exception cref="T:System.ObjectDisposedException">Thrown when the scanner has been closed.</exception>
	public Token[] ReadToEnd()
	{
		List<Token> TokenArray = new List<Token>();
		if (_Disposed)
		{
			throw new ObjectDisposedException("Scanner");
		}
		while (Peek().Type != TokenType.EndOfStream)
		{
			TokenArray.Add(Read());
		}
		return TokenArray.ToArray();
	}

	private int ParseInt(string literal)
	{
		int @base;
		if (literal.StartsWith("&"))
		{
			@base = ((literal[1] != 'H' && literal[1] != 'h') ? 8 : 16);
			literal = literal.Substring(2);
		}
		else
		{
			@base = 10;
		}
		return Convert.ToInt32(literal, @base);
	}
}