Initial commit

src/tokenizer.c

@@ -0,0 +1,116 @@
+#include "tokenizer.h"
+#include <ctype.h>
+#include <math.h>
+
+static StringView StringViewOfNumberTillNextNonDigit(StringView* source) {
+	StringView stringViewOfNumber = StringView_Slice(*source, 0, 0);
+	while (source->length != 0 && isdigit(source->source[0])) {
+		*source = StringView_Drop(*source, 1);
+		stringViewOfNumber.length ++;
+	}
+	return stringViewOfNumber;
+}
+
+static int64_t* _StringView_FoldInt64(char c, int64_t* i)
+{
+	*i = *i * 10 + (c - '0');
+	return i;
+}
+
+static double* _StringView_FoldDouble(char c, double* d)
+{
+	*d = *d * 10 + (c - '0');
+	return d;
+}
+
+static Token _Tokenizer_ParseInt64(bool negative, StringView integerPart)
+{
+	int64_t theInt64 = 0;
+	StringView_FoldLeft(integerPart, &theInt64, (StringViewFoldFunction) _StringView_FoldInt64);
+
+	return (Token) {
+		.type = TOKENTYPE_INTEGER,
+		.get = {
+			.integer = theInt64 * (negative ? -1 : 1)
+		}
+	};
+}
+
+static Token _Tokenizer_ParseDouble(bool negative, StringView integerPart, StringView decimalPart)
+{
+	double theDouble = 0.0;
+	StringView_FoldLeft(integerPart, &theDouble, (StringViewFoldFunction) _StringView_FoldDouble);
+	double theDecimal = 0.0;
+	StringView_FoldLeft(decimalPart, &theDecimal, (StringViewFoldFunction) _StringView_FoldDouble);
+
+	double result = (negative ? -1 : 1) * (theDouble + theDecimal / pow(10.0, decimalPart.length));
+
+	return (Token) {
+		.type = TOKENTYPE_DOUBLE,
+		.get = {
+			.decimal = result
+		}
+	};
+}
+
+static Token _Tokenizer_NumberToken(StringView* source)
+{
+	bool negative = false;
+	if (StringView_StartsWith(*source, StringView_FromString("-"))) {
+		negative = true;
+		*source = StringView_Drop(*source, 1);
+	}
+
+	StringView integerPart = StringViewOfNumberTillNextNonDigit(source);
+	bool has_point = false;
+	if (source->length != 0 && source->source[0] == '.') {
+		*source = StringView_Drop(*source, 1);
+		has_point = true;
+	}
+	StringView decimalPart = StringViewOfNumberTillNextNonDigit(source);
+
+	if (has_point) {
+		return _Tokenizer_ParseDouble(negative, integerPart, decimalPart);
+	} else {
+		return _Tokenizer_ParseInt64(negative, integerPart);
+	}
+}
+
+static bool _Tokenizer_IdentifierLetter(char c)
+{
+	return isalnum(c);
+}
+
+static Token _Tokenizer_IdentifierToken(StringView* source)
+{
+	StringView identifier = StringView_TakeWhile(*source, _Tokenizer_IdentifierLetter);
+	*source = StringView_Drop(*source, identifier.length);
+
+	return (Token) {
+		.type = TOKENTYPE_IDENTIFIER,
+		.get = {
+			.identifier = identifier,
+		}
+	};
+}
+
+Token Tokenizer_NextToken(StringView* source)
+{
+	while (source->length != 0 && isspace(source->source[0])) {
+		0[source] = StringView_Slice(*source, 1, source->length);
+	}
+
+	if (source->length == 0) {
+		return TOKEN_NONE;
+	}
+
+	if (isdigit(source->source[0]) || StringView_StartsWith(*source, StringView_FromString("-"))) {
+		// parse int/double
+		return _Tokenizer_NumberToken(source);
+	} else if (isalpha(source->source[0])) {
+		// parse name
+		return _Tokenizer_IdentifierToken(source);
+	} else {
+		return (Token) {.type = TOKENTYPE_ERROR, .get = {.error = *source } };
+	}
+}