Remove the TokenList structure
This struct was just a container for two pointers and just made things painful. This structure has been removed and the next pointer has been moved to the Token struct. Also, add a null check to the calling context after NewLexer().
This commit is contained in:
parent
9c172c5216
commit
7d8ed24eaf
5
lexer.c
5
lexer.c
|
@ -53,6 +53,7 @@ NewLexer(const char* filename)
|
|||
|
||||
state->rawFile[fileSize] = '\0';
|
||||
state->line = 1;
|
||||
state->column = 0;
|
||||
|
||||
readChar(state);
|
||||
return state;
|
||||
|
@ -159,6 +160,7 @@ newWhitespaceToken(Lexer* l)
|
|||
}
|
||||
tok->literal[count] = '\0';
|
||||
tok->length = count;
|
||||
tok->next = NULL;
|
||||
return tok;
|
||||
}
|
||||
|
||||
|
@ -272,6 +274,7 @@ newTickToken(Lexer* l)
|
|||
tok->literal = "```";
|
||||
tok->type = TT_TRIPLEBACKTICK;
|
||||
tok->length = 3;
|
||||
tok->next = NULL;
|
||||
readChar(l);
|
||||
readChar(l);
|
||||
return tok;
|
||||
|
@ -291,6 +294,7 @@ newToken(Lexer* l,
|
|||
tok->line = l->line;
|
||||
tok->column = l->column;
|
||||
tok->length = 1;
|
||||
tok->next = NULL;
|
||||
return tok;
|
||||
}
|
||||
|
||||
|
@ -306,5 +310,6 @@ newIdentToken(Lexer* l,
|
|||
tok->line = l->line;
|
||||
tok->column = l->column;
|
||||
tok->length = strlen(literal);
|
||||
tok->next = NULL;
|
||||
return tok;
|
||||
}
|
||||
|
|
33
main.c
33
main.c
|
@ -7,7 +7,7 @@
|
|||
#include "lexer.h"
|
||||
#include "node.h"
|
||||
|
||||
void writeTokenFile(TokenList* tl);
|
||||
void writeTokenFile(Token* startToken);
|
||||
|
||||
int
|
||||
main(int argc, const char** argv)
|
||||
|
@ -29,22 +29,31 @@ main(int argc, const char** argv)
|
|||
}
|
||||
|
||||
Lexer* l = NewLexer(argv[1]);
|
||||
TokenList* current = malloc(sizeof(TokenList));
|
||||
TokenList* tl = current;//= malloc(sizeof(TokenList));
|
||||
current->token = NULL;
|
||||
current->next = NULL;
|
||||
if (l == NULL) {
|
||||
printf("lexer borked\n");
|
||||
return 3;
|
||||
}
|
||||
|
||||
Token* current = NULL;
|
||||
Token* firstToken = NULL;
|
||||
|
||||
TokenType tt;
|
||||
do
|
||||
{
|
||||
Token* t = NextToken(l);
|
||||
tt = t->type;
|
||||
current = TokenListAdd(current, t);
|
||||
if (current != NULL) {
|
||||
current->next = t;
|
||||
}
|
||||
current = t;
|
||||
if (firstToken == NULL) {
|
||||
firstToken = t;
|
||||
}
|
||||
}
|
||||
while(tt != TT_EOF);
|
||||
|
||||
writeTokenFile(tl);
|
||||
Node* node = ParseNodes(tl);
|
||||
writeTokenFile(firstToken);
|
||||
Node* node = ParseNodes(firstToken);
|
||||
|
||||
printf("nodes:\n");
|
||||
while(node != NULL)
|
||||
|
@ -96,7 +105,7 @@ main(int argc, const char** argv)
|
|||
}
|
||||
|
||||
void
|
||||
writeTokenFile(TokenList* tl)
|
||||
writeTokenFile(Token* startToken)
|
||||
{
|
||||
printf("writeTokenFile() start\n");
|
||||
int count;
|
||||
|
@ -107,16 +116,16 @@ writeTokenFile(TokenList* tl)
|
|||
return;
|
||||
}
|
||||
|
||||
TokenList* current = tl;
|
||||
Token* current = startToken;
|
||||
for(count = 0; current != NULL; count++) {
|
||||
if (count == 0 && current->token == NULL)
|
||||
if (count == 0 && current == NULL)
|
||||
{
|
||||
printf("first token null\n");
|
||||
break;
|
||||
}
|
||||
|
||||
/*printf("writeTokenFile(): %s\n", TokenString(current->token));*/
|
||||
fprintf(fp, "%s\n", TokenString(current->token));
|
||||
fprintf(fp, "%s\n", TokenString(current));
|
||||
current = current->next;
|
||||
}
|
||||
fclose(fp);
|
||||
|
|
85
node.c
85
node.c
|
@ -7,16 +7,16 @@
|
|||
|
||||
static char stringBuff[STRING_BUFF_SIZE];
|
||||
|
||||
Node* parseHeader(TokenList** list);
|
||||
Node* parseCodeBlock(TokenList** list);
|
||||
Node* parseHeader(Token** firstToken);
|
||||
Node* parseCodeBlock(Token** firstToken);
|
||||
|
||||
Node*
|
||||
ParseNodes(TokenList* list)
|
||||
ParseNodes(Token* firstToken)
|
||||
{
|
||||
printf("ParseNodes() start\n");
|
||||
Node* first = NULL;
|
||||
Node* firstNode = NULL;
|
||||
|
||||
TokenList* currentToken = list;
|
||||
Token* currentToken = firstToken;
|
||||
Node* prevNode = NULL;
|
||||
|
||||
printf("ParseNodes() loop\n");
|
||||
|
@ -24,7 +24,7 @@ ParseNodes(TokenList* list)
|
|||
while (1) {
|
||||
Node* currentNode = NULL;
|
||||
|
||||
switch (currentToken->token->type) {
|
||||
switch (currentToken->type) {
|
||||
case TT_NEWLINE:
|
||||
break;
|
||||
case TT_HASH:
|
||||
|
@ -38,7 +38,7 @@ ParseNodes(TokenList* list)
|
|||
|
||||
case TT_EOF:
|
||||
printf("EOF found\n");
|
||||
return first;
|
||||
return firstNode;
|
||||
|
||||
default: // paragraph start?
|
||||
break;
|
||||
|
@ -57,64 +57,65 @@ ParseNodes(TokenList* list)
|
|||
prevNode->next = currentNode;
|
||||
}
|
||||
|
||||
if (first == NULL) {
|
||||
first = currentNode;
|
||||
if (firstNode == NULL) {
|
||||
firstNode = currentNode;
|
||||
}
|
||||
|
||||
prevNode = currentNode;
|
||||
}
|
||||
|
||||
return first;
|
||||
return firstNode;
|
||||
}
|
||||
|
||||
Node*
|
||||
parseHeader(TokenList** list)
|
||||
parseHeader(Token** startToken)
|
||||
{
|
||||
TokenList* l = *list;
|
||||
Token* t = *startToken;
|
||||
// Count the number of TT_HASH tokens
|
||||
int count = 1;
|
||||
while (l->next != NULL && l->next->token->type == TT_HASH)
|
||||
while (t->next != NULL && t->next->type == TT_HASH)
|
||||
{
|
||||
count++;
|
||||
l = l->next;
|
||||
t = t->next;
|
||||
}
|
||||
|
||||
if (l->next == NULL)
|
||||
if (t->next == NULL)
|
||||
{
|
||||
printf("Header missing text");
|
||||
return NULL;
|
||||
}
|
||||
l = l->next;
|
||||
t = t->next;
|
||||
|
||||
// Trim leading whitespace
|
||||
while (l->next != NULL && l->token->type == TT_WHITESPACE)
|
||||
while (t->next != NULL && t->type == TT_WHITESPACE)
|
||||
{
|
||||
l = l->next;
|
||||
t = t->next;
|
||||
}
|
||||
|
||||
if (l->next == NULL)
|
||||
if (t->next == NULL)
|
||||
{
|
||||
printf("Header missing text");
|
||||
return NULL;
|
||||
}
|
||||
|
||||
TokenList* end = l;
|
||||
Token* end = t;
|
||||
int len = 0;
|
||||
// find header text size
|
||||
while (end->token->type != TT_NEWLINE && end->token->type != TT_EOF) {
|
||||
len += end->token->length;
|
||||
while (end->type != TT_NEWLINE && end->type != TT_EOF) {
|
||||
len += end->length;
|
||||
end = end->next;
|
||||
}
|
||||
|
||||
char* strbuff = malloc(len+1);
|
||||
strbuff[0] = '\0';
|
||||
|
||||
while(l != end) {
|
||||
strncat(strbuff, l->token->literal, l->token->length);
|
||||
l = l->next;
|
||||
while(t != end) {
|
||||
strncat(strbuff, t->literal, t->length);
|
||||
t = t->next;
|
||||
}
|
||||
|
||||
*list = l;
|
||||
|
||||
*startToken = t;
|
||||
printf("header hash count: %d\ntext: '%s'\n", count, stringBuff);
|
||||
|
||||
HeaderNode* retval = malloc(sizeof(HeaderNode));
|
||||
|
@ -140,21 +141,21 @@ parseHeader(TokenList** list)
|
|||
}
|
||||
|
||||
Node*
|
||||
parseCodeBlock(TokenList** list)
|
||||
parseCodeBlock(Token** startToken)
|
||||
{
|
||||
TokenList* l = *list;
|
||||
// find closing ticks
|
||||
int tlen = 0; // number of tokens
|
||||
int clen = 0; // number of characters
|
||||
l = l->next; // skip past the opening triple backtick
|
||||
Token* t = *startToken;
|
||||
t = t->next; // skip past the opening triple backtick
|
||||
|
||||
// skip the first newline
|
||||
if (l->token->type == TT_NEWLINE) {
|
||||
l = l->next;
|
||||
if (t->type == TT_NEWLINE) {
|
||||
t = t->next;
|
||||
}
|
||||
|
||||
while (l->next != NULL && l->next->token->type != TT_TRIPLEBACKTICK) {
|
||||
if (l->next->token->type == TT_EOF) {
|
||||
while (t->next != NULL && t->type != TT_TRIPLEBACKTICK) {
|
||||
if (t->next->type == TT_EOF) {
|
||||
printf("premature EOF");
|
||||
|
||||
ErrorNode* err = malloc(sizeof(ErrorNode));
|
||||
|
@ -166,11 +167,11 @@ parseCodeBlock(TokenList** list)
|
|||
}
|
||||
|
||||
tlen++;
|
||||
clen += l->token->length;
|
||||
l = l->next;
|
||||
clen += t->length;
|
||||
t = t->next;
|
||||
}
|
||||
|
||||
l = *list;
|
||||
t = *startToken;
|
||||
|
||||
printf("codeblock token length: %d\n", tlen);
|
||||
printf("codeblock char length: %d\n", clen);
|
||||
|
@ -179,20 +180,20 @@ parseCodeBlock(TokenList** list)
|
|||
char* strbuff = malloc(sizeof(char)*clen+1);
|
||||
strbuff[0] = '\0';
|
||||
int i;
|
||||
l = l->next; // skip past the opening triple backtick
|
||||
t = t->next; // skip past the opening triple backtick
|
||||
|
||||
// skip the first newline
|
||||
if (l->token->type == TT_NEWLINE) {
|
||||
l = l->next;
|
||||
if (t->type == TT_NEWLINE) {
|
||||
t = t->next;
|
||||
}
|
||||
|
||||
for(i = 0; i < tlen; i++) {
|
||||
strncat(strbuff, l->token->literal, l->token->length);
|
||||
l = l->next;
|
||||
strncat(strbuff, t->literal, t->length);
|
||||
t = t->next;
|
||||
}
|
||||
|
||||
// skip past closing triple backtick
|
||||
*list = l->next;
|
||||
*startToken = t->next;
|
||||
|
||||
printf("malloc(%ld)\n", sizeof(CodeBlockNode));
|
||||
CodeBlockNode* ret = malloc(sizeof(CodeBlockNode));
|
||||
|
|
2
node.h
2
node.h
|
@ -62,7 +62,7 @@ typedef struct {
|
|||
} ParagraphNode;
|
||||
*/
|
||||
|
||||
Node* ParseNodes(TokenList* list);
|
||||
Node* ParseNodes(Token* firstToken);
|
||||
char* NodeTypeString(NodeType t);
|
||||
|
||||
#endif
|
||||
|
|
19
token.c
19
token.c
|
@ -10,25 +10,6 @@ static char stringBuff[STRING_BUFF_SIZE];
|
|||
|
||||
char* printableOnly(char* input);
|
||||
|
||||
TokenList*
|
||||
TokenListAdd(TokenList* current, Token* next)
|
||||
{
|
||||
if (current->token == NULL)
|
||||
{
|
||||
printf("current->token == null\n");
|
||||
current->token = next;
|
||||
return current;
|
||||
}
|
||||
|
||||
//printf("%s\n", TokenString(current->token));
|
||||
|
||||
TokenList* nl = malloc(sizeof(TokenList));
|
||||
nl->next = NULL;
|
||||
nl->token = next;
|
||||
current->next = nl;
|
||||
return nl;
|
||||
}
|
||||
|
||||
char*
|
||||
TokenString(Token* t)
|
||||
{
|
||||
|
|
8
token.h
8
token.h
|
@ -20,19 +20,13 @@ typedef enum {
|
|||
|
||||
typedef struct Token {
|
||||
TokenType type;
|
||||
struct Token* next;
|
||||
char* literal;
|
||||
int line;
|
||||
int column;
|
||||
int length;
|
||||
} Token;
|
||||
|
||||
typedef struct TokenList {
|
||||
Token* token;
|
||||
struct TokenList* next;
|
||||
} TokenList;
|
||||
|
||||
TokenList* TokenListAdd(TokenList* current, Token* next);
|
||||
|
||||
char* TokenString(Token* t);
|
||||
char* TokenTypeString(TokenType tt);
|
||||
|
||||
|
|
Loading…
Reference in New Issue