vegowotenks/flup
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Tokenizing and printing

Browse source
This commit is contained in:
vegowotenks 2024-10-01 10:53:26 +02:00
parent 371eca6269
commit b5912b95f7
5 changed files with 259 additions and 1 deletions
Download patch file Download diff file Expand all files Collapse all files

93
src/tokenizer.c
View file

@ -94,6 +94,42 @@ static Token _Tokenizer_IdentifierToken(StringView* source)
};
}
static Token _Tokenizer_SimpleToken(StringView* source)
{
const char* literal_table[] = { "{", "}", "&", ":", "+", "->", "-", "*", "/", "|", "==", "!=", "<", "<=", ">", ">=", "," };
const enum TokenType type_table[] = {
TOKENTYPE_LEFT_BRACE,
TOKENTYPE_RIGHT_BRACE,
TOKENTYPE_AMPERSAND,
TOKENTYPE_COLON,
TOKENTYPE_PLUS,
TOKENTYPE_ARROW,
TOKENTYPE_MINUS,
TOKENTYPE_MULTIPLY,
TOKENTYPE_DIVIDE,
TOKENTYPE_PIPE,
TOKENTYPE_EQUALITY,
TOKENTYPE_INEQUALITY,
TOKENTYPE_LESSTHAN,
TOKENTYPE_LESSEQUAL,
TOKENTYPE_GREATERTHAN,
TOKENTYPE_GREATEREQUAL,
TOKENTYPE_COMMA,
};
for (size_t i = 0; i < sizeof(literal_table) / sizeof(literal_table[0]); i++) {
StringView literal_view = StringView_FromString(literal_table[i]);
if (StringView_StartsWith(*source, literal_view)) {
*source = StringView_Drop(*source, literal_view.length);
return (Token) {
.type = type_table[i],
.get = { .identifier = STRINGVIEW_NONE }
};
}
}
return TOKEN_NONE;
}
Token Tokenizer_NextToken(StringView* source)
{
while (source->length != 0 && isspace(source->source[0])) {
@ -104,6 +140,13 @@ Token Tokenizer_NextToken(StringView* source)
return TOKEN_NONE;
}
{
Token simple_token = _Tokenizer_SimpleToken(source);
if (simple_token.type != TOKENTYPE_NONE) {
return simple_token;
}
}
if (isdigit(source->source[0]) || StringView_StartsWith(*source, StringView_FromString("-"))) {
// parse int/double
return _Tokenizer_NumberToken(source);
@ -114,3 +157,53 @@ Token Tokenizer_NextToken(StringView* source)
return (Token) {.type = TOKENTYPE_ERROR, .get = {.error = *source } };
}
}
const char* TokenType_ToString(enum TokenType type)
{
switch (type) {
case TOKENTYPE_NONE:
return "TOKENTYPE_NONE";
case TOKENTYPE_INTEGER:
return "TOKENTYPE_INTEGER";
case TOKENTYPE_DOUBLE:
return "TOKENTYPE_DOUBLE";
case TOKENTYPE_IDENTIFIER:
return "TOKENTYPE_IDENTIFIER";
case TOKENTYPE_LEFT_BRACE:
return "TOKENTYPE_LEFT_BRACE";
case TOKENTYPE_RIGHT_BRACE:
return "TOKENTYPE_RIGHT_BRACE";
case TOKENTYPE_AMPERSAND:
return "TOKENTYPE_AMPERSAND";
case TOKENTYPE_PLUS:
return "TOKENTYPE_PLUS";
case TOKENTYPE_MINUS:
return "TOKENTYPE_MINUS";
case TOKENTYPE_MULTIPLY:
return "TOKENTYPE_MULTIPLY";
case TOKENTYPE_DIVIDE:
return "TOKENTYPE_DIVIDE";
case TOKENTYPE_PIPE:
return "TOKENTYPE_PIPE";
case TOKENTYPE_ARROW:
return "TOKENTYPE_ARROW";
case TOKENTYPE_COLON:
return "TOKENTYPE_COLON";
case TOKENTYPE_ERROR:
return "TOKENTYPE_ERROR";
case TOKENTYPE_EQUALITY:
return "TOKENTYPE_EQUALITY";
case TOKENTYPE_INEQUALITY:
return "TOKENTYPE_INEQUALITY";
case TOKENTYPE_LESSTHAN:
return "TOKENTYPE_LESSTHAN";
case TOKENTYPE_LESSEQUAL:
return "TOKENTYPE_LESSEQUAL";
case TOKENTYPE_GREATERTHAN:
return "TOKENTYPE_GREATERTHAN";
case TOKENTYPE_GREATEREQUAL:
return "TOKENTYPE_GREATEREQUAL";
case TOKENTYPE_COMMA:
return "TOKENTYPE_COMMA";
}
}