Hi everyone,

I already posted an update yesterday where I mentioned that I have started work on the new parser which uses Flex and Bison to generate C code for parsing.

So far the experience has been very nice, I am able to test and develop things very fast compared to doing it all on my own.

I have completed the initial version of the tokenizer/scanner/lexer, it should be able to scan all tokens which could possibly occur in a valid script 🙂

%option noyywrap nodefault

%{
#include <stdlib.h>
#include <string.h>

char *token_str = NULL;
size_t token_len = 0;

size_t comment_level = 0;

//#define YY_USER_ACTION free(token_str); token_str = strndup(yytext, token_len = yyleng);
#define YY_USER_ACTION token_str = yytext; token_len = yyleng;

enum { // Token Types
	UNKNOWN = 420,
	WS,
	COMMENT, DIRECTIVE,
	NUMBER, STRING, BOOL,
	WORD,
	MACRO, VARIABLE,
	OPERATOR, BRACKET, DOT, COMMA,
};

%}

%x ML_COMMENT

WS	[ \t\r\n]+
NL	\r?\n?

DIGIT	[0-9]
XDIGIT	[0-9A-Fa-f]

QUOTE	[\'\"]

%%

 /* Whitespace */
{WS}	;

 /* Comment */
<INITIAL,ML_COMMENT>"#cs"|"#comment-start"{WS}	{BEGIN ML_COMMENT; ++comment_level;};
<ML_COMMENT>"#ce"|"#comment-end"	if (--comment_level == 0) BEGIN INITIAL;
<ML_COMMENT>(?s:.)	;
[;#][^\r\n]*	return *yytext == ';' ? COMMENT : DIRECTIVE;

 /* Number */
{DIGIT}+(\.{DIGIT}+(e{DIGIT}+)?)?	return NUMBER;
0[xX]{XDIGIT}+	return NUMBER;

 /* String */
\"[^\n\"]*\"	return STRING;
\'[^\n\']*\'	return STRING;

 /* Bool */
(?i:"True"|"False")	return BOOL;

 /* Word */
[A-Za-z][A-Za-z0-9]*	return WORD;

 /* Macro or Variable */
[@$][A-Za-z][A-Za-z0-9]*	return *yytext == '@' ? MACRO : VARIABLE;

 /* Operator */
[+\-*/^&=<>?:]	return OPERATOR;

 /* Misc */
[[\]()]	return BRACKET;
\.		return DOT;
\,		return COMMA;

 /* Catch-all for everything else */
.	return UNKNOWN;

%%

#include <stddef.h>
#include <stdio.h>

static void print_token(char *str, size_t len, int type) {
	puts("---### TOKEN ###---");
	char *token_type;
	switch (type) {
		case UNKNOWN:
			token_type = "Unknown";
			break;
		case WS:
			token_type = "Whitespace";
			break;
		case COMMENT:
			token_type = "Comment";
			break;
		case DIRECTIVE:
			token_type = "Directive";
			break;
		case NUMBER:
			token_type = "Number";
			break;
		case STRING:
			token_type = "String";
			break;
		case BOOL:
			token_type = "Boolean";
			break;
		case WORD:
			token_type = "Word";
			break;
		case MACRO:
			token_type = "Macro";
			break;
		case VARIABLE:
			token_type = "Variable";
			break;
		case OPERATOR:
			token_type = "Operator";
			break;
		case BRACKET:
			token_type = "Bracket";
			break;
		case DOT:
			token_type = "Dot";
			break;
		case COMMA:
			token_type = "Comma";
			break;
		default:
			token_type = "Unnamed";
			break;
	}
	fputs("Type: ", stdout);
	puts(token_type);
	fputs("Data: ", stdout);
	puts(str);
}

int main(void) {
	int type;
	for (;;) {
		type = yylex();
		if (!type) break;
		print_token(token_str, token_len, type);
	}
	return 0;
}

I passed this "script" as the input:

#cs
	#cs
		nested comment
	#ce
#ce

; Calm Mints

42
0xDEADBEEF
3.14
6.02214076e23

"Hell'o World"
'"To C or not to C"'

True
true
fAlSe
False
tRUe

@TheTruth
$TheFact

-((1 + 2) / 3) * 4

And here is the output:

---### TOKEN ###---
Type: Comment
Data: ; Calm Mints
---### TOKEN ###---
Type: Number
Data: 42
---### TOKEN ###---
Type: Number
Data: 0xDEADBEEF
---### TOKEN ###---
Type: Number
Data: 3.14
---### TOKEN ###---
Type: Number
Data: 6.02214076e23
---### TOKEN ###---
Type: String
Data: "Hell'o World"
---### TOKEN ###---
Type: String
Data: '"To C or not to C"'
---### TOKEN ###---
Type: Boolean
Data: True
---### TOKEN ###---
Type: Boolean
Data: true
---### TOKEN ###---
Type: Boolean
Data: fAlSe
---### TOKEN ###---
Type: Boolean
Data: False
---### TOKEN ###---
Type: Boolean
Data: tRUe
---### TOKEN ###---
Type: Macro
Data: @TheTruth
---### TOKEN ###---
Type: Variable
Data: $TheFact
---### TOKEN ###---
Type: Operator
Data: -
---### TOKEN ###---
Type: Bracket
Data: (
---### TOKEN ###---
Type: Bracket
Data: (
---### TOKEN ###---
Type: Number
Data: 1
---### TOKEN ###---
Type: Operator
Data: +
---### TOKEN ###---
Type: Number
Data: 2
---### TOKEN ###---
Type: Bracket
Data: )
---### TOKEN ###---
Type: Operator
Data: /
---### TOKEN ###---
Type: Number
Data: 3
---### TOKEN ###---
Type: Bracket
Data: )
---### TOKEN ###---
Type: Operator
Data: *
---### TOKEN ###---
Type: Number
Data: 4

Which is very similar if not exactly the same as what our old parser would have produced 🙂

The next step is to add support for including files (#include <...>) and then I will start work on the syntactic analysis, which is the actual "parsing" to convert list of words into actions. Hopefully I won't get held up by last time due to all the fatigue, using these tools should speed up the process significantly.

Stay tuned for more updates this week!