Lexical Analysis

هذا كود كتبته منذ فترة بلغة ال D

و هو قريب من السي++

Character

module IntelliDParser.Character;

/*
	This class represents a single character in the source text
	it holds information about the position of character and its value
*/
public class Character
{
	public const char EOF = '\0';

	public int sourceIndex, lineIndex,colIndex;
	public char value;

	this(char aval, int asrcIdx, int alnIdx, int aclIdx)
	{
		value = aval;
		sourceIndex = asrcIdx;
		lineIndex = alnIdx;
		colIndex = aclIdx;
	}

}

Token

module IntelliDParser.Token;
import IntelliDParser.Character;
public class Token
{
	public enum
	{
		NONE,
		COMMENT,
		IDENTIFIER
	}

	public Character[] tokenValue;
	public int lineIndex, colIndex;
	public int type;

	this(Character c)
	{
		tokenValue = new Character[1];
		tokenValue[0] = c;
	}
	public void addChar(Character c)
	{
		tokenValue.length = tokenValue.length + 1;
		tokenValue[tokenValue.length-1] = c;
	}
	public char[] toString()
	{
		char[] arr = new char[1];
		arr[0] = tokenValue[0].value;
		for(int i = 1; i<tokenValue.length; i++)
		{
			arr.length = arr.length + 1;
			arr = tokenValue.value;
		}
		return arr;
	}
}

Scanner:

module IntelliDParser.Scanner;
import IntelliDParser.Character;
public class Scanner
{
	char[] sourceText;
	private int srcIndex, colIndex, lineIndex,lastIndex;
	this(char[] arg_sT)
	{
		sourceText = arg_sT;
		lineIndex = 0;
		srcIndex = -1;
		colIndex = -1;
		lastIndex = sourceText.length - 1;
	}
	public Character next()
	{
		srcIndex++;
		if(srcIndex > 0 && srcIndex <= lastIndex)
		{
			if(sourceText[srcIndex] == '\n')
			{
				lineIndex++;
				colIndex = -1;
			}
		}
		colIndex++;

		if(srcIndex > lastIndex)
		{
			return new Character(Character.EOF,srcIndex,lineIndex,colIndex);
		}
		else
		{
			return new Character(sourceText[srcIndex],srcIndex,lineIndex,colIndex);
		}

	}
	public Character topNext()
	{
		int li = lineIndex, ci = colIndex;
		if(srcIndex+1 > 0 && srcIndex+1 <= lastIndex)
		{
			if(sourceText[srcIndex+1] == '\n')
			{
				li++;
				ci = -1;
			}
		}
		ci++;

		if(srcIndex > lastIndex)
		{
			return new Character(Character.EOF,srcIndex+1,li,ci);
		}
		else
		{
			return new Character(sourceText[srcIndex+1],srcIndex+1,li,ci);
		}
	}
}

Symbols:

module IntelliDParser.Symbols;
public bool containsB(char a, char[][] b)
{
	return true;
}

char[][] arrKeywords =
[
"abstract",
"alias",
"align",
"asm",
"assert",
"auto",
"body",
"break",
"case",
"cast",
"catch",
"class",
"const",
"continue",
"debug",
"default",
"delegate",
"delete",
"deprecated",
"do",
"else",
"enum",
"export",
"extern",
"false",
"final",
"finally",
"for",
"foreach",
"foreach_reverse",
"function",
"goto",
"if",
"import",
"in",
"inout",
"interface",
"invariant",
"is",
"lazy",
"mixin",
"module",
"new",
"null",
"out",
"override",
"package",
"pragma",
"private",
"protected",
"public",
"return",
"scope",
"static",
"struct",
"super",
"switch",
"synchronized",
"template",
"this",
"throw",
"true",
"try",
"typedef",
"typeid",
"typeof",
"union",
"unittest",
"version",
"volatile",
"while",
"with"
];


char[][] arrDataTypes =
[
"bool",
"byte",
"cdouble",
"cent",
"cfloat",
"char",
"creal",
"dchar",
"double",
"float",
"idouble",
"ifloat",
"int",
"ireal",
"long",
"real",
"short",
"ubyte",
"ucent",
"uint",
"ulong",
"ushort",
"void",
"wchar"
];

char[][] arrOperators =
[
"/",
"/=",
".",
"..",
"...",
"&",
"&=",
"&&",
"|",
"|=",
"||",
"-",
"-=",
"--",
"+",
"+=",
"++",
"<",
"<=",
"<<",
"<<=",
"<>",
"<>=",
">",
">=",
">>=",
">>>=",
">>",
">>>",
"!",
"!=",
"!==",
"!<>",
"!<>=",
"!<",
"!<=",
"!>",
"!>=",
"!~",
"(",
")",
"[",
"]",
"{",
"}",
"?",
",",
";",
":",
"$",
"=",
"==",
"===",
"*",
"*=",
"%",
"%=",
"^",
"^=",
"~",
"~=",
"~~",
];

public char[] arrWhiteSpace = " \t\r\n";

char[] arrStringChars = "'\"";

char[] arrCommentChars = "///*/+";
char[] arrCommentCloseChars = "  */+/";

Lexer:

module IntelliDParser.Lexer;

import IntelliDParser.Scanner;
import IntelliDParser.Token;
import IntelliDParser.Character;
import IntelliDParser.Symbols;

import tango.text.Util;
import tango.text.convert.Integer;
import dwt.widgets.MessageBox;
import dwt.DWT;

public class Lexer
{
	char[] sourceText;
	Scanner scanner;
	Character curChar;
	this(char[] asrcTxt)
	{
		sourceText = asrcTxt;
		scanner = new Scanner(sourceText);
		curChar = scanner.next();
	}
	public Token nextToken()
	{
		char[] index = format(new char[2],curChar.sourceIndex);
		char[] a = " index: " ~ index;
		MessageBox.showMessageBox("curChar: " ~ curChar.value ~ "" ~ a,"a",null,DWT.OK);
		//process whitespaces
		while(arrWhiteSpace.contains(curChar.value))
		{
			curChar = scanner.next();
		}

		//process comments
		if(arrCommentChars.containsPattern("" ~ curChar.value ~ scanner.topNext().value))
		{
			int posOfCom = arrCommentChars.locatePattern("" ~ curChar.value ~ scanner.topNext().value);
			char[] closeCom = arrCommentCloseChars[posOfCom .. posOfCom+2];
			Token token = new Token(curChar);
			token.addChar(scanner.next());
			token.type = Token.COMMENT;
			curChar = scanner.next();
			while(!matching(closeCom.ptr,("" ~ curChar.value ~ scanner.topNext().value).ptr,2u))
			{
				token.addChar(curChar);
				curChar = scanner.next();
			}
			MessageBox.showMessageBox("After: " ~ curChar.value ~ scanner.topNext().value,"a",null,DWT.OK);
			token.addChar(curChar);
			curChar = scanner.next();
			token.addChar(curChar);
			curChar = scanner.next();
			return token;
		}
		return null;
	}
}

طبعا يجب معرفة المبدأ من خلال دراسة مادة الكومبايلر

http://www.google.jo/search?q=writing+a+le...lient=firefox-a