Zorchenhimer
/
readme-thing
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Remove the TokenList structure 

This struct was just a container for two pointers and just made things
painful.  This structure has been removed and the next pointer has been
moved to the Token struct.

Also, add a null check to the calling context after NewLexer().
This commit is contained in:
Zorchenhimer 2023-10-22 16:50:18 -04:00
parent 9c172c5216
commit 7d8ed24eaf
6 changed files with 71 additions and 81 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
lexer.c
View File

@ -53,6 +53,7 @@ NewLexer(const char* filename)
state->rawFile[fileSize] = '\0';
state->line = 1;
state->column = 0;
readChar(state);
return state;
@ -159,6 +160,7 @@ newWhitespaceToken(Lexer* l)
}
tok->literal[count] = '\0';
tok->length = count;
tok->next = NULL;
return tok;
}
@ -272,6 +274,7 @@ newTickToken(Lexer* l)
tok->literal = "```";
tok->type = TT_TRIPLEBACKTICK;
tok->length = 3;
tok->next = NULL;
readChar(l);
readChar(l);
return tok;
@ -291,6 +294,7 @@ newToken(Lexer* l,
tok->line = l->line;
tok->column = l->column;
tok->length = 1;
tok->next = NULL;
return tok;
}
@ -306,5 +310,6 @@ newIdentToken(Lexer* l,
tok->line = l->line;
tok->column = l->column;
tok->length = strlen(literal);
tok->next = NULL;
return tok;
}

33
main.c
View File

@ -7,7 +7,7 @@
#include "lexer.h"
#include "node.h"
void writeTokenFile(TokenList* tl);
void writeTokenFile(Token* startToken);
int
main(int argc, const char** argv)
@ -29,22 +29,31 @@ main(int argc, const char** argv)
}
Lexer* l = NewLexer(argv[1]);
TokenList* current = malloc(sizeof(TokenList));
TokenList* tl = current;//= malloc(sizeof(TokenList));
current->token = NULL;
current->next = NULL;
if (l == NULL) {
printf("lexer borked\n");
return 3;
}
Token* current = NULL;
Token* firstToken = NULL;
TokenType tt;
do
{
Token* t = NextToken(l);
tt = t->type;
current = TokenListAdd(current, t);
if (current != NULL) {
current->next = t;
}
current = t;
if (firstToken == NULL) {
firstToken = t;
}
}
while(tt != TT_EOF);
writeTokenFile(tl);
Node* node = ParseNodes(tl);
writeTokenFile(firstToken);
Node* node = ParseNodes(firstToken);
printf("nodes:\n");
while(node != NULL)
@ -96,7 +105,7 @@ main(int argc, const char** argv)
}
void
writeTokenFile(TokenList* tl)
writeTokenFile(Token* startToken)
{
printf("writeTokenFile() start\n");
int count;
@ -107,16 +116,16 @@ writeTokenFile(TokenList* tl)
return;
}
TokenList* current = tl;
Token* current = startToken;
for(count = 0; current != NULL; count++) {
if (count == 0 && current->token == NULL)
if (count == 0 && current == NULL)
{
printf("first token null\n");
break;
}
/*printf("writeTokenFile(): %s\n", TokenString(current->token));*/
fprintf(fp, "%s\n", TokenString(current->token));
fprintf(fp, "%s\n", TokenString(current));
current = current->next;
}
fclose(fp);

85
node.c
View File

@ -7,16 +7,16 @@
static char stringBuff[STRING_BUFF_SIZE];
Node* parseHeader(TokenList** list);
Node* parseCodeBlock(TokenList** list);
Node* parseHeader(Token** firstToken);
Node* parseCodeBlock(Token** firstToken);
Node*
ParseNodes(TokenList* list)
ParseNodes(Token* firstToken)
{
printf("ParseNodes() start\n");
Node* first = NULL;
Node* firstNode = NULL;
TokenList* currentToken = list;
Token* currentToken = firstToken;
Node* prevNode = NULL;
printf("ParseNodes() loop\n");
@ -24,7 +24,7 @@ ParseNodes(TokenList* list)
while (1) {
Node* currentNode = NULL;
switch (currentToken->token->type) {
switch (currentToken->type) {
case TT_NEWLINE:
break;
case TT_HASH:
@ -38,7 +38,7 @@ ParseNodes(TokenList* list)
case TT_EOF:
printf("EOF found\n");
return first;
return firstNode;
default: // paragraph start?
break;
@ -57,64 +57,65 @@ ParseNodes(TokenList* list)
prevNode->next = currentNode;
}
if (first == NULL) {
first = currentNode;
if (firstNode == NULL) {
firstNode = currentNode;
}
prevNode = currentNode;
}
return first;
return firstNode;
}
Node*
parseHeader(TokenList** list)
parseHeader(Token** startToken)
{
TokenList* l = *list;
Token* t = *startToken;
// Count the number of TT_HASH tokens
int count = 1;
while (l->next != NULL && l->next->token->type == TT_HASH)
while (t->next != NULL && t->next->type == TT_HASH)
{
count++;
l = l->next;
t = t->next;
}
if (l->next == NULL)
if (t->next == NULL)
{
printf("Header missing text");
return NULL;
}
l = l->next;
t = t->next;
// Trim leading whitespace
while (l->next != NULL && l->token->type == TT_WHITESPACE)
while (t->next != NULL && t->type == TT_WHITESPACE)
{
l = l->next;
t = t->next;
}
if (l->next == NULL)
if (t->next == NULL)
{
printf("Header missing text");
return NULL;
}
TokenList* end = l;
Token* end = t;
int len = 0;
// find header text size
while (end->token->type != TT_NEWLINE && end->token->type != TT_EOF) {
len += end->token->length;
while (end->type != TT_NEWLINE && end->type != TT_EOF) {
len += end->length;
end = end->next;
}
char* strbuff = malloc(len+1);
strbuff[0] = '\0';
while(l != end) {
strncat(strbuff, l->token->literal, l->token->length);
l = l->next;
while(t != end) {
strncat(strbuff, t->literal, t->length);
t = t->next;
}
*list = l;
*startToken = t;
printf("header hash count: %d\ntext: '%s'\n", count, stringBuff);
HeaderNode* retval = malloc(sizeof(HeaderNode));
@ -140,21 +141,21 @@ parseHeader(TokenList** list)
}
Node*
parseCodeBlock(TokenList** list)
parseCodeBlock(Token** startToken)
{
TokenList* l = *list;
// find closing ticks
int tlen = 0; // number of tokens
int clen = 0; // number of characters
l = l->next; // skip past the opening triple backtick
Token* t = *startToken;
t = t->next; // skip past the opening triple backtick
// skip the first newline
if (l->token->type == TT_NEWLINE) {
l = l->next;
if (t->type == TT_NEWLINE) {
t = t->next;
}
while (l->next != NULL && l->next->token->type != TT_TRIPLEBACKTICK) {
if (l->next->token->type == TT_EOF) {
while (t->next != NULL && t->type != TT_TRIPLEBACKTICK) {
if (t->next->type == TT_EOF) {
printf("premature EOF");
ErrorNode* err = malloc(sizeof(ErrorNode));
@ -166,11 +167,11 @@ parseCodeBlock(TokenList** list)
}
tlen++;
clen += l->token->length;
l = l->next;
clen += t->length;
t = t->next;
}
l = *list;
t = *startToken;
printf("codeblock token length: %d\n", tlen);
printf("codeblock char length: %d\n", clen);
@ -179,20 +180,20 @@ parseCodeBlock(TokenList** list)
char* strbuff = malloc(sizeof(char)*clen+1);
strbuff[0] = '\0';
int i;
l = l->next; // skip past the opening triple backtick
t = t->next; // skip past the opening triple backtick
// skip the first newline
if (l->token->type == TT_NEWLINE) {
l = l->next;
if (t->type == TT_NEWLINE) {
t = t->next;
}
for(i = 0; i < tlen; i++) {
strncat(strbuff, l->token->literal, l->token->length);
l = l->next;
strncat(strbuff, t->literal, t->length);
t = t->next;
}
// skip past closing triple backtick
*list = l->next;
*startToken = t->next;
printf("malloc(%ld)\n", sizeof(CodeBlockNode));
CodeBlockNode* ret = malloc(sizeof(CodeBlockNode));

2
node.h
View File

@ -62,7 +62,7 @@ typedef struct {
} ParagraphNode;
*/
Node* ParseNodes(TokenList* list);
Node* ParseNodes(Token* firstToken);
char* NodeTypeString(NodeType t);
#endif

19
token.c
View File

@ -10,25 +10,6 @@ static char stringBuff[STRING_BUFF_SIZE];
char* printableOnly(char* input);
TokenList*
TokenListAdd(TokenList* current, Token* next)
{
if (current->token == NULL)
{
printf("current->token == null\n");
current->token = next;
return current;
}
//printf("%s\n", TokenString(current->token));
TokenList* nl = malloc(sizeof(TokenList));
nl->next = NULL;
nl->token = next;
current->next = nl;
return nl;
}
char*
TokenString(Token* t)
{

8
token.h
View File

@ -20,19 +20,13 @@ typedef enum {
typedef struct Token {
TokenType type;
struct Token* next;
char* literal;
int line;
int column;
int length;
} Token;
typedef struct TokenList {
Token* token;
struct TokenList* next;
} TokenList;
TokenList* TokenListAdd(TokenList* current, Token* next);
char* TokenString(Token* t);
char* TokenTypeString(TokenType tt);