Initial commit

src/tokenizer.h

@@ -0,0 +1,32 @@
+#ifndef FLUP_TOKENIZER_H
+#define FLUP_TOKENIZER_H
+
+#include "../include/utilitiec/StringView/StringView.h"
+#include <stdint.h>
+
+enum TokenType {
+	TOKENTYPE_NONE,
+	TOKENTYPE_INTEGER,
+	TOKENTYPE_DOUBLE,
+	TOKENTYPE_IDENTIFIER,
+	TOKENTYPE_ERROR,
+};
+
+union TokenContent {
+	StringView identifier;
+	StringView error;
+	int64_t    integer;
+	double     decimal;
+};
+
+typedef struct Token_s {
+	enum TokenType type;
+	union TokenContent get;
+} Token;
+
+
+#define TOKEN_NONE ((Token) {.type = TOKENTYPE_NONE, .get = {.error = STRINGVIEW_NONE } } )
+
+Token Tokenizer_NextToken(StringView* source);
+
+#endif //header guard