progress

AspClassic.Scripting/Runtime/TokenizerService.cs

@@ -0,0 +1,50 @@
+using System.Collections.Generic;
+using System.IO;
+
+namespace AspClassic.Scripting.Runtime;
+
+public abstract class TokenizerService
+{
+	public abstract object CurrentState { get; }
+
+	public abstract SourceLocation CurrentPosition { get; }
+
+	public abstract bool IsRestartable { get; }
+
+	public abstract ErrorSink ErrorSink { get; set; }
+
+	public abstract void Initialize(object state, TextReader sourceReader, SourceUnit sourceUnit, SourceLocation initialLocation);
+
+	public abstract TokenInfo ReadToken();
+
+	public virtual bool SkipToken()
+	{
+		return ReadToken().Category != TokenCategory.EndOfStream;
+	}
+
+	public virtual IEnumerable<TokenInfo> ReadTokens(int countOfChars)
+	{
+		List<TokenInfo> list = new List<TokenInfo>();
+		int index = CurrentPosition.Index;
+		while (CurrentPosition.Index - index < countOfChars)
+		{
+			TokenInfo item = ReadToken();
+			if (item.Category == TokenCategory.EndOfStream)
+			{
+				break;
+			}
+			list.Add(item);
+		}
+		return list;
+	}
+
+	public bool SkipTokens(int countOfChars)
+	{
+		bool result = false;
+		int index = CurrentPosition.Index;
+		while (CurrentPosition.Index - index < countOfChars && (result = SkipToken()))
+		{
+		}
+		return result;
+	}
+}