Merge pull request #11 from adamhutchings/token

Add unlexing capability

src/lexer/lex.c

@@ -0,0 +1,26 @@
+#include "lex.h"
+
+#include <string.h> // memcpy
+
+int lex(Lexer *l, Token *t) {
+    // If there are any tokens waiting in the putback buffer, read from there
+    // instead.
+    if (l->unlexed_count > 0) {
+        l->unlexed_count--;
+        memcpy(t, &l->unlexed[l->unlexed_count], sizeof(Token));
+        return 0;
+    }
+    // Now, do the actual lexing: TODO!
+    return 0;
+}
+
+int unlex(Lexer *l, Token *t) {
+    // First, make sure we can actually fit it in the buffer.
+    if (l->unlexed_count >= TOKEN_PUTBACKS) {
+        // TODO: error reporting
+        return -1; // Error return code
+    }
+    memcpy(&l->unlexed[l->unlexed_count], t, sizeof(Token));
+    l->unlexed_count++;
+    return 0;
+}

src/lexer/lex.h

@@ -4,13 +4,20 @@
 
 #include "token.h"
 
+// The maximum number of tokens we can put back.
+#define TOKEN_PUTBACKS 5
+
 // This is a lexer structure, which will contain all of the information about
-// the state of a lexer. For now all we need is the file pointer which actually
-// has the data we need, but in the future we will need more.
+// the state of a lexer.
 typedef struct {
-    FILE * fp;
+    FILE * fp; // The file we are reading from.
+    Token unlexed[TOKEN_PUTBACKS];
+    unsigned unlexed_count;
 } Lexer;
 
 // Takes an open file pointer, and a token as input. It fills the token struct
 // with the next available token from the file.
 int lex(Lexer * l, Token * token);
+
+// Put a token back to be lexed again in the future.
+int unlex(Lexer * l, Token * token);