vegowotenks/flup
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

File splitting, object type

Browse source
This commit is contained in:
vegowotenks 2024-10-10 15:54:22 +02:00
parent 5313cc5603
commit 3f05fa5141
14 changed files with 194 additions and 15 deletions
Download patch file Download diff file Expand all files Collapse all files

42
src/tokenizer.c
View file

@ -94,6 +94,16 @@ static Token _Tokenizer_IdentifierToken(StringView* source)
};
}
static bool _Tokenizer_ContinueCommentFunction(char c)
{
return c != '\n';
}
static bool _Tokenizer_ContinueStringFunction(char c)
{
return c != '"';
}
static Token _Tokenizer_SimpleToken(StringView* source)
{
const char* literal_table[] = { "{", "}", "&", ":", "+", "->", "-", "*", "/", "|", "==", "!=", "<", "<=", ">", ">=", ",", ";", "bind", "as", "(", ")" };
@ -135,6 +145,30 @@ static Token _Tokenizer_SimpleToken(StringView* source)
return TOKEN_NONE;
}
Token _Tokenizer_CommentToken(StringView* source)
{
StringView comment = StringView_SpanWhile(source, _Tokenizer_ContinueCommentFunction);
return (Token) {
.type = TOKENTYPE_COMMENT,
.get = { .identifier = comment },
};
}
Token _Tokenizer_StringToken(StringView* source)
{
*source = StringView_Drop(*source, 1);
StringView string = StringView_SpanWhile(source, _Tokenizer_ContinueStringFunction);
string.source--;
string.length += 2;
*source = StringView_Drop(*source, 1);
return (Token) {
.type = TOKENTYPE_STRING,
.get = { .identifier = string },
};
}
Token Tokenizer_NextToken(StringView* source)
{
while (source->length != 0 && isspace(source->source[0])) {
@ -158,6 +192,10 @@ Token Tokenizer_NextToken(StringView* source)
} else if (isalpha(source->source[0])) {
// parse name
return _Tokenizer_IdentifierToken(source);
} else if (StringView_StartsWith(*source, StringView_FromString("#"))) {
return _Tokenizer_CommentToken(source);
} else if (StringView_StartsWith(*source, StringView_FromString("\""))) {
return _Tokenizer_StringToken(source);
} else {
return (Token) {.type = TOKENTYPE_ERROR, .get = {.error = *source } };
}
@ -220,6 +258,10 @@ const char* TokenType_ToString(enum TokenType type)
return "LEFT_PAREN";
case TOKENTYPE_RIGHT_PAREN:
return "RIGHT_PAREN";
case TOKENTYPE_COMMENT:
return "COMMENT";
case TOKENTYPE_STRING:
return "STRING";
}
return "INVALID";