Zorchenhimer
/
readme-thing
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Remove the TokenList structure 

This struct was just a container for two pointers and just made things
painful.  This structure has been removed and the next pointer has been
moved to the Token struct.

Also, add a null check to the calling context after NewLexer().
This commit is contained in:
Zorchenhimer 2023-10-22 16:50:18 -04:00
parent 9c172c5216
commit 7d8ed24eaf
6 changed files with 71 additions and 81 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
lexer.c
View File

@ -53,6 +53,7 @@ NewLexer(const char* filename)
state->rawFile[fileSize] = '\0'; state->rawFile[fileSize] = '\0';
state->line = 1; state->line = 1;
state->column = 0;
readChar(state); readChar(state);
return state; return state;
@ -159,6 +160,7 @@ newWhitespaceToken(Lexer* l)
} }
tok->literal[count] = '\0'; tok->literal[count] = '\0';
tok->length = count; tok->length = count;
tok->next = NULL;
return tok; return tok;
} }
@ -272,6 +274,7 @@ newTickToken(Lexer* l)
tok->literal = "```"; tok->literal = "```";
tok->type = TT_TRIPLEBACKTICK; tok->type = TT_TRIPLEBACKTICK;
tok->length = 3; tok->length = 3;
tok->next = NULL;
readChar(l); readChar(l);
readChar(l); readChar(l);
return tok; return tok;
@ -291,6 +294,7 @@ newToken(Lexer* l,
tok->line = l->line; tok->line = l->line;
tok->column = l->column; tok->column = l->column;
tok->length = 1; tok->length = 1;
tok->next = NULL;
return tok; return tok;
} }
@ -306,5 +310,6 @@ newIdentToken(Lexer* l,
tok->line = l->line; tok->line = l->line;
tok->column = l->column; tok->column = l->column;
tok->length = strlen(literal); tok->length = strlen(literal);
tok->next = NULL;
return tok; return tok;
} }

33
main.c
View File

@ -7,7 +7,7 @@
#include "lexer.h" #include "lexer.h"
#include "node.h" #include "node.h"
void writeTokenFile(TokenList* tl); void writeTokenFile(Token* startToken);
int int
main(int argc, const char** argv) main(int argc, const char** argv)
@ -29,22 +29,31 @@ main(int argc, const char** argv)
} }
Lexer* l = NewLexer(argv[1]); Lexer* l = NewLexer(argv[1]);
TokenList* current = malloc(sizeof(TokenList)); if (l == NULL) {
TokenList* tl = current;//= malloc(sizeof(TokenList)); printf("lexer borked\n");
current->token = NULL; return 3;
current->next = NULL; }
Token* current = NULL;
Token* firstToken = NULL;
TokenType tt; TokenType tt;
do do
{ {
Token* t = NextToken(l); Token* t = NextToken(l);
tt = t->type; tt = t->type;
current = TokenListAdd(current, t); if (current != NULL) {
current->next = t;
}
current = t;
if (firstToken == NULL) {
firstToken = t;
}
} }
while(tt != TT_EOF); while(tt != TT_EOF);
writeTokenFile(tl); writeTokenFile(firstToken);
Node* node = ParseNodes(tl); Node* node = ParseNodes(firstToken);
printf("nodes:\n"); printf("nodes:\n");
while(node != NULL) while(node != NULL)
@ -96,7 +105,7 @@ main(int argc, const char** argv)
} }
void void
writeTokenFile(TokenList* tl) writeTokenFile(Token* startToken)
{ {
printf("writeTokenFile() start\n"); printf("writeTokenFile() start\n");
int count; int count;
@ -107,16 +116,16 @@ writeTokenFile(TokenList* tl)
return; return;
} }
TokenList* current = tl; Token* current = startToken;
for(count = 0; current != NULL; count++) { for(count = 0; current != NULL; count++) {
if (count == 0 && current->token == NULL) if (count == 0 && current == NULL)
{ {
printf("first token null\n"); printf("first token null\n");
break; break;
} }
/*printf("writeTokenFile(): %s\n", TokenString(current->token));*/ /*printf("writeTokenFile(): %s\n", TokenString(current->token));*/
fprintf(fp, "%s\n", TokenString(current->token)); fprintf(fp, "%s\n", TokenString(current));
current = current->next; current = current->next;
} }
fclose(fp); fclose(fp);

85
node.c
View File

@ -7,16 +7,16 @@
static char stringBuff[STRING_BUFF_SIZE]; static char stringBuff[STRING_BUFF_SIZE];
Node* parseHeader(TokenList** list); Node* parseHeader(Token** firstToken);
Node* parseCodeBlock(TokenList** list); Node* parseCodeBlock(Token** firstToken);
Node* Node*
ParseNodes(TokenList* list) ParseNodes(Token* firstToken)
{ {
printf("ParseNodes() start\n"); printf("ParseNodes() start\n");
Node* first = NULL; Node* firstNode = NULL;
TokenList* currentToken = list; Token* currentToken = firstToken;
Node* prevNode = NULL; Node* prevNode = NULL;
printf("ParseNodes() loop\n"); printf("ParseNodes() loop\n");
@ -24,7 +24,7 @@ ParseNodes(TokenList* list)
while (1) { while (1) {
Node* currentNode = NULL; Node* currentNode = NULL;
switch (currentToken->token->type) { switch (currentToken->type) {
case TT_NEWLINE: case TT_NEWLINE:
break; break;
case TT_HASH: case TT_HASH:
@ -38,7 +38,7 @@ ParseNodes(TokenList* list)
case TT_EOF: case TT_EOF:
printf("EOF found\n"); printf("EOF found\n");
return first; return firstNode;
default: // paragraph start? default: // paragraph start?
break; break;
@ -57,64 +57,65 @@ ParseNodes(TokenList* list)
prevNode->next = currentNode; prevNode->next = currentNode;
} }
if (first == NULL) { if (firstNode == NULL) {
first = currentNode; firstNode = currentNode;
} }
prevNode = currentNode; prevNode = currentNode;
} }
return first; return firstNode;
} }
Node* Node*
parseHeader(TokenList** list) parseHeader(Token** startToken)
{ {
TokenList* l = *list; Token* t = *startToken;
// Count the number of TT_HASH tokens // Count the number of TT_HASH tokens
int count = 1; int count = 1;
while (l->next != NULL && l->next->token->type == TT_HASH) while (t->next != NULL && t->next->type == TT_HASH)
{ {
count++; count++;
l = l->next; t = t->next;
} }
if (l->next == NULL) if (t->next == NULL)
{ {
printf("Header missing text"); printf("Header missing text");
return NULL; return NULL;
} }
l = l->next; t = t->next;
// Trim leading whitespace // Trim leading whitespace
while (l->next != NULL && l->token->type == TT_WHITESPACE) while (t->next != NULL && t->type == TT_WHITESPACE)
{ {
l = l->next; t = t->next;
} }
if (l->next == NULL) if (t->next == NULL)
{ {
printf("Header missing text"); printf("Header missing text");
return NULL; return NULL;
} }
TokenList* end = l; Token* end = t;
int len = 0; int len = 0;
// find header text size // find header text size
while (end->token->type != TT_NEWLINE && end->token->type != TT_EOF) { while (end->type != TT_NEWLINE && end->type != TT_EOF) {
len += end->token->length; len += end->length;
end = end->next; end = end->next;
} }
char* strbuff = malloc(len+1); char* strbuff = malloc(len+1);
strbuff[0] = '\0'; strbuff[0] = '\0';
while(l != end) { while(t != end) {
strncat(strbuff, l->token->literal, l->token->length); strncat(strbuff, t->literal, t->length);
l = l->next; t = t->next;
} }
*list = l;
*startToken = t;
printf("header hash count: %d\ntext: '%s'\n", count, stringBuff); printf("header hash count: %d\ntext: '%s'\n", count, stringBuff);
HeaderNode* retval = malloc(sizeof(HeaderNode)); HeaderNode* retval = malloc(sizeof(HeaderNode));
@ -140,21 +141,21 @@ parseHeader(TokenList** list)
} }
Node* Node*
parseCodeBlock(TokenList** list) parseCodeBlock(Token** startToken)
{ {
TokenList* l = *list;
// find closing ticks // find closing ticks
int tlen = 0; // number of tokens int tlen = 0; // number of tokens
int clen = 0; // number of characters int clen = 0; // number of characters
l = l->next; // skip past the opening triple backtick Token* t = *startToken;
t = t->next; // skip past the opening triple backtick
// skip the first newline // skip the first newline
if (l->token->type == TT_NEWLINE) { if (t->type == TT_NEWLINE) {
l = l->next; t = t->next;
} }
while (l->next != NULL && l->next->token->type != TT_TRIPLEBACKTICK) { while (t->next != NULL && t->type != TT_TRIPLEBACKTICK) {
if (l->next->token->type == TT_EOF) { if (t->next->type == TT_EOF) {
printf("premature EOF"); printf("premature EOF");
ErrorNode* err = malloc(sizeof(ErrorNode)); ErrorNode* err = malloc(sizeof(ErrorNode));
@ -166,11 +167,11 @@ parseCodeBlock(TokenList** list)
} }
tlen++; tlen++;
clen += l->token->length; clen += t->length;
l = l->next; t = t->next;
} }
l = *list; t = *startToken;
printf("codeblock token length: %d\n", tlen); printf("codeblock token length: %d\n", tlen);
printf("codeblock char length: %d\n", clen); printf("codeblock char length: %d\n", clen);
@ -179,20 +180,20 @@ parseCodeBlock(TokenList** list)
char* strbuff = malloc(sizeof(char)*clen+1); char* strbuff = malloc(sizeof(char)*clen+1);
strbuff[0] = '\0'; strbuff[0] = '\0';
int i; int i;
l = l->next; // skip past the opening triple backtick t = t->next; // skip past the opening triple backtick
// skip the first newline // skip the first newline
if (l->token->type == TT_NEWLINE) { if (t->type == TT_NEWLINE) {
l = l->next; t = t->next;
} }
for(i = 0; i < tlen; i++) { for(i = 0; i < tlen; i++) {
strncat(strbuff, l->token->literal, l->token->length); strncat(strbuff, t->literal, t->length);
l = l->next; t = t->next;
} }
// skip past closing triple backtick // skip past closing triple backtick
*list = l->next; *startToken = t->next;
printf("malloc(%ld)\n", sizeof(CodeBlockNode)); printf("malloc(%ld)\n", sizeof(CodeBlockNode));
CodeBlockNode* ret = malloc(sizeof(CodeBlockNode)); CodeBlockNode* ret = malloc(sizeof(CodeBlockNode));

2
node.h
View File

@ -62,7 +62,7 @@ typedef struct {
} ParagraphNode; } ParagraphNode;
*/ */
Node* ParseNodes(TokenList* list); Node* ParseNodes(Token* firstToken);
char* NodeTypeString(NodeType t); char* NodeTypeString(NodeType t);
#endif #endif

19
token.c
View File

@ -10,25 +10,6 @@ static char stringBuff[STRING_BUFF_SIZE];
char* printableOnly(char* input); char* printableOnly(char* input);
TokenList*
TokenListAdd(TokenList* current, Token* next)
{
if (current->token == NULL)
{
printf("current->token == null\n");
current->token = next;
return current;
}
//printf("%s\n", TokenString(current->token));
TokenList* nl = malloc(sizeof(TokenList));
nl->next = NULL;
nl->token = next;
current->next = nl;
return nl;
}
char* char*
TokenString(Token* t) TokenString(Token* t)
{ {

8
token.h
View File

@ -20,19 +20,13 @@ typedef enum {
typedef struct Token { typedef struct Token {
TokenType type; TokenType type;
struct Token* next;
char* literal; char* literal;
int line; int line;
int column; int column;
int length; int length;
} Token; } Token;
typedef struct TokenList {
Token* token;
struct TokenList* next;
} TokenList;
TokenList* TokenListAdd(TokenList* current, Token* next);
char* TokenString(Token* t); char* TokenString(Token* t);
char* TokenTypeString(TokenType tt); char* TokenTypeString(TokenType tt);