diff options
-rw-r--r-- | evaluate.c | 63 | ||||
-rw-r--r-- | expression.c | 55 | ||||
-rw-r--r-- | expression.h | 16 | ||||
-rw-r--r-- | lib.c | 26 | ||||
-rw-r--r-- | lib.h | 13 | ||||
-rw-r--r-- | parse.c | 100 | ||||
-rw-r--r-- | pre-process.c | 130 | ||||
-rw-r--r-- | show-parse.c | 16 | ||||
-rw-r--r-- | symbol.c | 10 | ||||
-rw-r--r-- | symbol.h | 6 | ||||
-rw-r--r-- | test-lexing.c | 4 | ||||
-rw-r--r-- | token.h | 14 | ||||
-rw-r--r-- | tokenize.c | 85 |
13 files changed, 273 insertions, 265 deletions
@@ -27,7 +27,7 @@ static int evaluate_symbol(struct expression *expr) struct symbol *base_type; if (!sym) { - warn(expr->token, "undefined identifier '%s'", show_token(expr->token)); + warn(expr->pos, "undefined identifier '%s'", show_ident(expr->symbol_name)); return 0; } examine_symbol_type(sym); @@ -112,7 +112,7 @@ static int get_int_value(struct expression *expr, const char *str) if (BITS_IN_LONG == BITS_IN_INT) modifiers = MOD_LONG | MOD_UNSIGNED; } - warn(expr->token, "value is so big it is%s%s%s", + warn(expr->pos, "value is so big it is%s%s%s", (modifiers & MOD_UNSIGNED) ? " unsigned":"", (modifiers & MOD_LONG) ? " long":"", (modifiers & MOD_LONGLONG) ? " long":""); @@ -128,7 +128,7 @@ static int evaluate_constant(struct expression *expr) { struct token *token = expr->token; - switch (token->type) { + switch (token_type(token)) { case TOKEN_INTEGER: return get_int_value(expr, token->integer); @@ -137,11 +137,13 @@ static int evaluate_constant(struct expression *expr) expr->ctype = &int_ctype; expr->value = (char) token->character; return 1; + case TOKEN_STRING: expr->ctype = &string_ctype; return 1; + default: - warn(token, "non-typed expression"); + warn(expr->pos, "non-typed expression"); } return 0; } @@ -212,7 +214,7 @@ static int cast_value(struct expression *expr, struct symbol *newtype, static struct expression * cast_to(struct expression *old, struct symbol *type) { - struct expression *expr = alloc_expression(old->token, EXPR_CAST); + struct expression *expr = alloc_expression(old->pos, EXPR_CAST); expr->ctype = type; expr->cast_type = type; expr->cast_expression = old; @@ -233,7 +235,7 @@ static int is_int_type(struct symbol *type) static int bad_expr_type(struct expression *expr) { - warn(expr->token, "incompatible types for operation"); + warn(expr->pos, "incompatible types for operation"); return 0; } @@ -274,8 +276,10 @@ static int evaluate_int_binop(struct expression *expr) static struct symbol *degenerate(struct expression *expr, struct symbol *ctype) { if (ctype->type == SYM_ARRAY) { - struct symbol *sym = alloc_symbol(expr->token, SYM_PTR); + struct symbol *sym = alloc_symbol(expr->pos, SYM_PTR); sym->ctype = ctype->ctype; + sym->bit_size = BITS_IN_POINTER; + sym->alignment = POINTER_ALIGNMENT; ctype = sym; } return ctype; @@ -298,8 +302,8 @@ static int evaluate_ptr_add(struct expression *expr, struct expression *ptr, str expr->ctype = degenerate(expr, ptr_type); if (ctype->bit_size > BITS_IN_CHAR) { struct expression *add = expr; - struct expression *mul = alloc_expression(expr->token, EXPR_BINOP); - struct expression *val = alloc_expression(expr->token, EXPR_VALUE); + struct expression *mul = alloc_expression(expr->pos, EXPR_BINOP); + struct expression *val = alloc_expression(expr->pos, EXPR_VALUE); val->ctype = size_t_ctype; val->value = ctype->bit_size >> 3; @@ -310,7 +314,6 @@ static int evaluate_ptr_add(struct expression *expr, struct expression *ptr, str mul->right = val; /* Leave 'add->op' as 'expr->op' - either '+' or '-' */ - add->ctype = ptr_type; add->left = ptr; add->right = mul; } @@ -360,16 +363,16 @@ static int evaluate_ptr_sub(struct expression *expr, struct expression *l, struc ctype = same_ptr_types(ltype, rtype); if (!ctype) { - warn(expr->token, "subtraction of different types can't work"); + warn(expr->pos, "subtraction of different types can't work"); return 0; } examine_symbol_type(ctype); expr->ctype = ssize_t_ctype; if (ctype->bit_size > BITS_IN_CHAR) { - struct expression *sub = alloc_expression(expr->token, EXPR_BINOP); + struct expression *sub = alloc_expression(expr->pos, EXPR_BINOP); struct expression *div = expr; - struct expression *val = alloc_expression(expr->token, EXPR_VALUE); + struct expression *val = alloc_expression(expr->pos, EXPR_VALUE); val->ctype = size_t_ctype; val->value = ctype->bit_size >> 3; @@ -489,19 +492,19 @@ static int evaluate_preop(struct expression *expr) case '*': if (ctype->type != SYM_PTR && ctype->type != SYM_ARRAY) { - warn(expr->token, "cannot derefence this type"); + warn(expr->pos, "cannot derefence this type"); return 0; } examine_symbol_type(expr->ctype); expr->ctype = ctype->ctype.base_type; if (!expr->ctype) { - warn(expr->token, "undefined type"); + warn(expr->pos, "undefined type"); return 0; } return 1; case '&': { - struct symbol *symbol = alloc_symbol(expr->token, SYM_PTR); + struct symbol *symbol = alloc_symbol(expr->pos, SYM_PTR); symbol->ctype.base_type = ctype; symbol->bit_size = BITS_IN_POINTER; symbol->alignment = POINTER_ALIGNMENT; @@ -538,7 +541,7 @@ struct symbol *find_identifier(struct ident *ident, struct symbol_list *_list, i for (i = 0; i < list->nr; i++) { struct symbol *sym = (struct symbol *) list->list[i]; if (sym->ident) { - if (sym->ident->ident != ident) + if (sym->ident != ident) continue; *offset = sym->offset; return sym; @@ -575,29 +578,29 @@ static int evaluate_dereference(struct expression *expr) if (expr->op == SPECIAL_DEREFERENCE) { /* Arrays will degenerate into pointers for '->' */ if (ctype->type != SYM_PTR && ctype->type != SYM_ARRAY) { - warn(expr->token, "expected a pointer to a struct/union"); + warn(expr->pos, "expected a pointer to a struct/union"); return 0; } ctype = ctype->ctype.base_type; } if (!ctype || (ctype->type != SYM_STRUCT && ctype->type != SYM_UNION)) { - warn(expr->token, "expected structure or union"); + warn(expr->pos, "expected structure or union"); return 0; } offset = 0; member = find_identifier(token->ident, ctype->symbol_list, &offset); if (!member) { - warn(expr->token, "no such struct/union member"); + warn(expr->pos, "no such struct/union member"); return 0; } add = deref; if (offset != 0) { - add = alloc_expression(expr->token, EXPR_BINOP); + add = alloc_expression(expr->pos, EXPR_BINOP); add->op = '+'; add->ctype = &ptr_ctype; add->left = deref; - add->right = alloc_expression(expr->token, EXPR_VALUE); + add->right = alloc_expression(expr->pos, EXPR_VALUE); add->right->ctype = &int_ctype; add->right->value = offset; } @@ -632,7 +635,7 @@ static int evaluate_sizeof(struct expression *expr) size = expr->cast_expression->ctype->bit_size; } if (size & 7) { - warn(expr->token, "cannot size expression"); + warn(expr->pos, "cannot size expression"); return 0; } expr->type = EXPR_VALUE; @@ -694,15 +697,15 @@ static int evaluate_call(struct expression *expr) if (ctype->type == SYM_PTR || ctype->type == SYM_ARRAY) ctype = ctype->ctype.base_type; if (ctype->type != SYM_FN) { - warn(expr->token, "not a function"); + warn(expr->pos, "not a function"); return 0; } args = expression_list_size(expr->args); fnargs = symbol_list_size(ctype->arguments); if (args < fnargs) - warn(expr->token, "not enough arguments for function"); + warn(expr->pos, "not enough arguments for function"); if (args > fnargs && !ctype->variadic) - warn(expr->token, "too many arguments for function"); + warn(expr->pos, "too many arguments for function"); expr->ctype = ctype->ctype.base_type; return 1; } @@ -774,12 +777,12 @@ long long get_expression_value(struct expression *expr) if (expr->cast_type) { examine_symbol_type(expr->cast_type); if (expr->cast_type->bit_size & 7) { - warn(expr->token, "type has no size"); + warn(expr->pos, "type has no size"); return 0; } return expr->cast_type->bit_size >> 3; } - warn(expr->token, "expression sizes not yet supported"); + warn(expr->pos, "expression sizes not yet supported"); return 0; case EXPR_CONSTANT: evaluate_constant(expr); @@ -789,7 +792,7 @@ long long get_expression_value(struct expression *expr) case EXPR_SYMBOL: { struct symbol *sym = expr->symbol; if (!sym || !sym->ctype.base_type || sym->ctype.base_type->type != SYM_ENUM) { - warn(expr->token, "undefined identifier in constant expression"); + warn(expr->pos, "undefined identifier '%s' in constant expression", expr->symbol_name); return 0; } return sym->value; @@ -836,6 +839,6 @@ long long get_expression_value(struct expression *expr) default: break; } - error(expr->token, "bad constant expression"); + error(expr->pos, "bad constant expression"); return 0; } diff --git a/expression.c b/expression.c index 62f196a..2062f88 100644 --- a/expression.c +++ b/expression.c @@ -40,7 +40,7 @@ struct token *parens_expression(struct token *token, struct expression **expr, c { token = expect(token, '(', where); if (match_op(token, '{')) { - struct expression *e = alloc_expression(token, EXPR_STATEMENT); + struct expression *e = alloc_expression(token->pos, EXPR_STATEMENT); struct statement *stmt = alloc_statement(token, STMT_COMPOUND); *expr = e; e->statement = stmt; @@ -57,31 +57,34 @@ struct token *primary_expression(struct token *token, struct expression **tree) { struct expression *expr = NULL; - switch (token->type) { + switch (token_type(token)) { case TOKEN_INTEGER: case TOKEN_FP: case TOKEN_CHAR: - expr = alloc_expression(token, EXPR_CONSTANT); + expr = alloc_expression(token->pos, EXPR_CONSTANT); + expr->token = token; token = token->next; break; case TOKEN_IDENT: { - expr = alloc_expression(token, EXPR_SYMBOL); + expr = alloc_expression(token->pos, EXPR_SYMBOL); + expr->symbol_name = token->ident; expr->symbol = lookup_symbol(token->ident, NS_SYMBOL); token = token->next; break; } case TOKEN_STRING: - expr = alloc_expression(token, EXPR_CONSTANT); + expr = alloc_expression(token->pos, EXPR_CONSTANT); + expr->token = token; do { token = token->next; - } while (token->type == TOKEN_STRING); + } while (token_type(token) == TOKEN_STRING); break; case TOKEN_SPECIAL: if (token->special == '(') { - expr = alloc_expression(token, EXPR_PREOP); + expr = alloc_expression(token->pos, EXPR_PREOP); expr->op = '('; token = parens_expression(token, &expr->unop, "in expression"); break; @@ -113,11 +116,11 @@ static struct token *postfix_expression(struct token *token, struct expression * struct expression *expr = NULL; token = primary_expression(token, &expr); - while (expr && token->type == TOKEN_SPECIAL) { + while (expr && token_type(token) == TOKEN_SPECIAL) { switch (token->special) { case '[': { /* Array dereference */ - struct expression *deref = alloc_expression(token, EXPR_PREOP); - struct expression *add = alloc_expression(token, EXPR_BINOP); + struct expression *deref = alloc_expression(token->pos, EXPR_PREOP); + struct expression *add = alloc_expression(token->pos, EXPR_BINOP); deref->op = '*'; deref->unop = add; @@ -131,7 +134,7 @@ static struct token *postfix_expression(struct token *token, struct expression * } case SPECIAL_INCREMENT: /* Post-increment */ case SPECIAL_DECREMENT: { /* Post-decrement */ - struct expression *post = alloc_expression(token, EXPR_POSTOP); + struct expression *post = alloc_expression(token->pos, EXPR_POSTOP); post->op = token->special; post->unop = expr; expr = post; @@ -140,12 +143,12 @@ static struct token *postfix_expression(struct token *token, struct expression * } case '.': /* Structure member dereference */ case SPECIAL_DEREFERENCE: { /* Structure pointer member dereference */ - struct expression *deref = alloc_expression(token, EXPR_DEREF); + struct expression *deref = alloc_expression(token->pos, EXPR_DEREF); deref->op = token->special; deref->deref = expr; token = token->next; - if (token->type != TOKEN_IDENT) { - warn(token, "Expected member name"); + if (token_type(token) != TOKEN_IDENT) { + warn(token->pos, "Expected member name"); break; } deref->member = token; @@ -155,7 +158,7 @@ static struct token *postfix_expression(struct token *token, struct expression * } case '(': { /* Function call */ - struct expression *call = alloc_expression(token, EXPR_CALL); + struct expression *call = alloc_expression(token->pos, EXPR_CALL); call->op = '('; call->fn = expr; token = expression_list(token->next, &call->args); @@ -176,10 +179,10 @@ static struct token *postfix_expression(struct token *token, struct expression * static struct token *cast_expression(struct token *token, struct expression **tree); static struct token *unary_expression(struct token *token, struct expression **tree) { - if (token->type == TOKEN_IDENT && + if (token_type(token) == TOKEN_IDENT && (token->ident == &sizeof_ident || token->ident == &__alignof___ident)) { - struct expression *sizeof_ex = alloc_expression(token, EXPR_SIZEOF); + struct expression *sizeof_ex = alloc_expression(token->pos, EXPR_SIZEOF); *tree = sizeof_ex; tree = &sizeof_ex->unop; token = token->next; @@ -189,11 +192,11 @@ static struct token *unary_expression(struct token *token, struct expression **t return expect(token, ')', "at end of sizeof type-name"); } - if (token->type == TOKEN_SPECIAL) { + if (token_type(token) == TOKEN_SPECIAL) { if (match_oplist(token->special, SPECIAL_INCREMENT, SPECIAL_DECREMENT, '&', '*', '+', '-', '~', '!', 0)) { - struct expression *unary = alloc_expression(token, EXPR_PREOP); + struct expression *unary = alloc_expression(token->pos, EXPR_PREOP); unary->op = token->special; *tree = unary; return cast_expression(token->next, &unary->unop); @@ -213,7 +216,7 @@ static struct token *cast_expression(struct token *token, struct expression **tr if (match_op(token, '(')) { struct token *next = token->next; if (lookup_type(next)) { - struct expression *cast = alloc_expression(next, EXPR_CAST); + struct expression *cast = alloc_expression(next->pos, EXPR_CAST); struct symbol *sym; token = typename(next, &sym); @@ -237,7 +240,7 @@ static struct token *lr_binop_expression(struct token *token, struct expression struct token * next = inner(token, &left); if (left) { - while (next->type == TOKEN_SPECIAL) { + while (token_type(next) == TOKEN_SPECIAL) { struct expression *top, *right = NULL; int op = next->special; va_list args; @@ -251,10 +254,10 @@ static struct token *lr_binop_expression(struct token *token, struct expression break; } va_end(args); - top = alloc_expression(next, type); + top = alloc_expression(next->pos, type); next = inner(next->next, &right); if (!right) { - warn(next, "No right hand side of '%s'-expression", show_special(op)); + warn(next->pos, "No right hand side of '%s'-expression", show_special(op)); break; } top->op = op; @@ -322,7 +325,7 @@ struct token *conditional_expression(struct token *token, struct expression **tr { token = logical_or_expression(token, tree); if (match_op(token, '?')) { - struct expression *expr = alloc_expression(token, EXPR_CONDITIONAL); + struct expression *expr = alloc_expression(token->pos, EXPR_CONDITIONAL); expr->op = token->special; expr->left = *tree; *tree = expr; @@ -336,7 +339,7 @@ struct token *conditional_expression(struct token *token, struct expression **tr struct token *assignment_expression(struct token *token, struct expression **tree) { token = conditional_expression(token, tree); - if (token->type == TOKEN_SPECIAL) { + if (token_type(token) == TOKEN_SPECIAL) { static const int assignments[] = { '=', SPECIAL_ADD_ASSIGN, SPECIAL_MINUS_ASSIGN, SPECIAL_TIMES_ASSIGN, SPECIAL_DIV_ASSIGN, @@ -346,7 +349,7 @@ struct token *assignment_expression(struct token *token, struct expression **tre int i, op = token->special; for (i = 0; i < sizeof(assignments)/sizeof(int); i++) if (assignments[i] == op) { - struct expression * expr = alloc_expression(token, EXPR_ASSIGNMENT); + struct expression * expr = alloc_expression(token->pos, EXPR_ASSIGNMENT); expr->left = *tree; expr->op = op; *tree = expr; diff --git a/expression.h b/expression.h index bf201ba..1897153 100644 --- a/expression.h +++ b/expression.h @@ -32,9 +32,12 @@ enum expression_type { struct expression { enum expression_type type; int op; - struct token *token; + struct position pos; struct symbol *ctype; union { + // EXPR_CONSTANT + struct token *token; + // EXPR_VALUE unsigned long long value; @@ -42,7 +45,10 @@ struct expression { struct expression *unop; // EXPR_SYMBOL - struct symbol *symbol; + struct symbol_arg { + struct symbol *symbol; + struct ident *symbol_name; + }; // EXPR_STATEMENT struct statement *statement; @@ -90,11 +96,11 @@ struct token *assignment_expression(struct token *token, struct expression **tre extern int evaluate_expression(struct expression *); -static inline struct expression *alloc_expression(struct token *token, int type) +static inline struct expression *alloc_expression(struct position pos, int type) { struct expression *expr = __alloc_expression(0); expr->type = type; - expr->token = token; + expr->pos = pos; return expr; } @@ -103,7 +109,7 @@ struct token *typename(struct token *, struct symbol **); static inline int lookup_type(struct token *token) { - if (token->type == TOKEN_IDENT) + if (token->pos.type == TOKEN_IDENT) return lookup_symbol(token->ident, NS_TYPEDEF) != NULL; return 0; } @@ -28,8 +28,8 @@ struct token *expect(struct token *token, int op, const char *where) static struct token bad_token; if (token != &bad_token) { bad_token.next = token; - warn(token, "Expected %s %s", show_special(op), where); - warn(token, "got %s", show_token(token)); + warn(token->pos, "Expected %s %s", show_special(op), where); + warn(token->pos, "got %s", show_token(token)); } if (op == ';') return skip_to(token, op); @@ -223,27 +223,19 @@ void add_ptr_list(struct ptr_list **listp, void *ptr) list->nr = nr; } -static void do_warn(const char *type, struct token *token, const char * fmt, va_list args) +static void do_warn(const char *type, struct position pos, const char * fmt, va_list args) { static char buffer[512]; const char *name; - int pos,line; vsprintf(buffer, fmt, args); - name = "EOF"; - pos = 0; - line = 0; - if (token) { - name = input_streams[token->stream].name; - pos = token->pos; - line = token->line; - } + name = input_streams[pos.stream].name; fprintf(stderr, "%s: %s:%d:%d: %s\n", - type, name, line, pos, buffer); + type, name, pos.line, pos.pos, buffer); } -void warn(struct token *token, const char * fmt, ...) +void warn(struct position pos, const char * fmt, ...) { static int warnings = 0; @@ -257,16 +249,16 @@ void warn(struct token *token, const char * fmt, ...) va_list args; va_start(args, fmt); - do_warn("warning", token, fmt, args); + do_warn("warning", pos, fmt, args); va_end(args); warnings++; } -void error(struct token *token, const char * fmt, ...) +void error(struct position pos, const char * fmt, ...) { va_list args; va_start(args, fmt); - do_warn("error", token, fmt, args); + do_warn("error", pos, fmt, args); va_end(args); exit(1); } @@ -8,6 +8,15 @@ extern unsigned int hexval(unsigned int c); +struct position { + unsigned int type:6, + stream:10, + pos:14, + newline:1, + whitespace:1; + unsigned int line; +}; + struct ident; struct token; struct symbol; @@ -19,8 +28,8 @@ struct expression_list; struct token *skip_to(struct token *, int); struct token *expect(struct token *, int, const char *); -extern void warn(struct token *, const char *, ...); -extern void error(struct token *, const char *, ...); +extern void warn(struct position, const char *, ...); +extern void error(struct position, const char *, ...); #define __DECLARE_ALLOCATOR(type, x) \ extern type *__alloc_##x(int); \ @@ -30,9 +30,9 @@ struct statement *alloc_statement(struct token * token, int type) static struct token *struct_declaration_list(struct token *token, struct symbol_list **list); -static struct symbol * indirect(struct token *token, struct ctype *ctype, int type) +static struct symbol * indirect(struct position pos, struct ctype *ctype, int type) { - struct symbol *sym = alloc_symbol(token, type); + struct symbol *sym = alloc_symbol(pos, type); sym->ctype.base_type = ctype->base_type; sym->ctype.modifiers = ctype->modifiers & ~MOD_STORAGE; @@ -46,7 +46,8 @@ static struct symbol *lookup_or_create_symbol(enum namespace ns, enum type type, { struct symbol *sym = lookup_symbol(token->ident, ns); if (!sym) { - sym = alloc_symbol(token, type); + sym = alloc_symbol(token->pos, type); + sym->ident = token->ident; bind_symbol(sym, token->ident, ns); } return sym; @@ -59,7 +60,7 @@ struct token *struct_union_enum_specifier(enum namespace ns, enum type type, struct symbol *sym; ctype->modifiers = 0; - if (token->type == TOKEN_IDENT) { + if (token_type(token) == TOKEN_IDENT) { sym = lookup_or_create_symbol(ns, type, token); token = token->next; ctype->base_type = sym; @@ -72,12 +73,12 @@ struct token *struct_union_enum_specifier(enum namespace ns, enum type type, // private struct/union/enum type if (!match_op(token, '{')) { - warn(token, "expected declaration"); + warn(token->pos, "expected declaration"); ctype->base_type = &bad_type; return token; } - sym = alloc_symbol(token, type); + sym = alloc_symbol(token->pos, type); token = parse(token->next, sym); ctype->base_type = sym; return expect(token, '}', "at end of specifier"); @@ -96,11 +97,11 @@ struct token *struct_or_union_specifier(enum type type, struct token *token, str static struct token *parse_enum_declaration(struct token *token, struct symbol *parent) { int nextval = 0; - while (token->type == TOKEN_IDENT) { + while (token_type(token) == TOKEN_IDENT) { struct token *next = token->next; struct symbol *sym; - sym = alloc_symbol(token, SYM_NODE); + sym = alloc_symbol(token->pos, SYM_NODE); bind_symbol(sym, token->ident, NS_SYMBOL); sym->ctype.base_type = parent; @@ -131,7 +132,7 @@ struct token *typeof_specifier(struct token *token, struct ctype *ctype) struct expression *expr; if (!match_op(token, '(')) { - warn(token, "expected '(' after typeof"); + warn(token->pos, "expected '(' after typeof"); return token; } if (lookup_type(token->next)) { @@ -183,7 +184,7 @@ static struct token *type_qualifiers(struct token *next, struct ctype *ctype) unsigned long mod; next = token->next; - if (token->type != TOKEN_IDENT) + if (token_type(token) != TOKEN_IDENT) break; s = lookup_symbol(token->ident, NS_TYPEDEF); if (!s) @@ -247,7 +248,7 @@ static struct token *declaration_specifiers(struct token *next, struct ctype *ct unsigned long mod; next = token->next; - if (token->type != TOKEN_IDENT) + if (token_type(token) != TOKEN_IDENT) break; ident = token->ident; @@ -274,7 +275,7 @@ static struct token *declaration_specifiers(struct token *next, struct ctype *ct if (type) { if (type != ctype->base_type) { if (ctype->base_type) { - warn(token, "Strange mix of types"); + warn(token->pos, "Strange mix of types"); continue; } ctype->base_type = type; @@ -291,7 +292,7 @@ static struct token *declaration_specifiers(struct token *next, struct ctype *ct } dup = (mod & old) | (extra & old) | (extra & mod); if (dup) - warn(token, "Just how %s do you want this type to be?", + warn(token->pos, "Just how %s do you want this type to be?", modifier_string(dup)); ctype->modifiers = old | mod | extra; } @@ -327,14 +328,14 @@ static struct token *abstract_array_declarator(struct token *token, struct symbo } static struct token *parameter_type_list(struct token *, struct symbol *); -static struct token *declarator(struct token *token, struct symbol **tree, struct token **p); +static struct token *declarator(struct token *token, struct symbol **tree, struct ident **p); -static struct token *direct_declarator(struct token *token, struct symbol **tree, struct token **p) +static struct token *direct_declarator(struct token *token, struct symbol **tree, struct ident **p) { struct ctype *ctype = &(*tree)->ctype; - if (p && token->type == TOKEN_IDENT) { - *p = token; + if (p && token_type(token) == TOKEN_IDENT) { + *p = token->ident; token = token->next; } @@ -344,7 +345,7 @@ static struct token *direct_declarator(struct token *token, struct symbol **tree token = attribute_specifier(token->next, &ctype); continue; } - if (token->type != TOKEN_SPECIAL) + if (token_type(token) != TOKEN_SPECIAL) return token; /* @@ -369,13 +370,13 @@ static struct token *direct_declarator(struct token *token, struct symbol **tree continue; } - sym = indirect(token, ctype, SYM_FN); + sym = indirect(token->pos, ctype, SYM_FN); token = parameter_type_list(next, sym); token = expect(token, ')', "in function declarator"); continue; } if (token->special == '[') { - struct symbol *array = indirect(token, ctype, SYM_ARRAY); + struct symbol *array = indirect(token->pos, ctype, SYM_ARRAY); token = abstract_array_declarator(token->next, array); token = expect(token, ']', "in abstract_array_declarator"); continue; @@ -383,7 +384,7 @@ static struct token *direct_declarator(struct token *token, struct symbol **tree if (token->special == ':') { struct symbol *bitfield; struct expression *expr; - bitfield = indirect(token, ctype, SYM_BITFIELD); + bitfield = indirect(token->pos, ctype, SYM_BITFIELD); token = conditional_expression(token->next, &expr); bitfield->fieldwidth = get_expression_value(expr); continue; @@ -391,7 +392,6 @@ static struct token *direct_declarator(struct token *token, struct symbol **tree break; } if (p) { - (*tree)->token = *p; (*tree)->ident = *p; } return token; @@ -406,7 +406,7 @@ static struct token *pointer(struct token *token, struct ctype *ctype) base_type = ctype->base_type; while (match_op(token,'*')) { - struct symbol *ptr = alloc_symbol(NULL, SYM_PTR); + struct symbol *ptr = alloc_symbol(token->pos, SYM_PTR); ptr->ctype.modifiers = modifiers & ~MOD_STORAGE; ptr->ctype.base_type = base_type; @@ -420,7 +420,7 @@ static struct token *pointer(struct token *token, struct ctype *ctype) return token; } -static struct token *declarator(struct token *token, struct symbol **tree, struct token **p) +static struct token *declarator(struct token *token, struct symbol **tree, struct ident **p) { token = pointer(token, &(*tree)->ctype); return direct_declarator(token, tree, p); @@ -433,8 +433,8 @@ static struct token *struct_declaration_list(struct token *token, struct symbol_ token = declaration_specifiers(token, &ctype); for (;;) { - struct token *ident = NULL; - struct symbol *decl = alloc_symbol(token, SYM_NODE); + struct ident *ident = NULL; + struct symbol *decl = alloc_symbol(token->pos, SYM_NODE); decl->ctype = ctype; token = pointer(token, &decl->ctype); token = direct_declarator(token, &decl, &ident); @@ -456,12 +456,12 @@ static struct token *struct_declaration_list(struct token *token, struct symbol_ static struct token *parameter_declaration(struct token *token, struct symbol **tree) { - struct token *ident = NULL; + struct ident *ident = NULL; struct symbol *sym; struct ctype ctype = { 0, }; token = declaration_specifiers(token, &ctype); - sym = alloc_symbol(token, SYM_NODE); + sym = alloc_symbol(token->pos, SYM_NODE); sym->ctype = ctype; *tree = sym; token = pointer(token, &sym->ctype); @@ -471,7 +471,7 @@ static struct token *parameter_declaration(struct token *token, struct symbol ** struct token *typename(struct token *token, struct symbol **p) { - struct symbol *sym = alloc_symbol(token, SYM_NODE); + struct symbol *sym = alloc_symbol(token->pos, SYM_NODE); *p = sym; token = declaration_specifiers(token, &sym->ctype); return declarator(token, &sym, NULL); @@ -524,12 +524,12 @@ struct token *statement(struct token *token, struct statement **tree) struct statement *stmt = alloc_statement(token, STMT_NONE); *tree = stmt; - if (token->type == TOKEN_IDENT) { + if (token_type(token) == TOKEN_IDENT) { if (token->ident == &if_ident) { stmt->type = STMT_IF; token = parens_expression(token->next, &stmt->if_conditional, "after if"); token = statement(token, &stmt->if_true); - if (token->type != TOKEN_IDENT) + if (token_type(token) != TOKEN_IDENT) return token; if (token->ident != &else_ident) return token; @@ -606,10 +606,10 @@ default_statement: struct statement *iterator; token = statement(token->next, &iterator); - if (token->type == TOKEN_IDENT && token->ident == &while_ident) + if (token_type(token) == TOKEN_IDENT && token->ident == &while_ident) token = token->next; else - warn(token, "expected 'while' after 'do'"); + warn(token->pos, "expected 'while' after 'do'"); token = parens_expression(token, &expr, "after 'do-while'"); stmt->type = STMT_ITERATOR; @@ -621,18 +621,18 @@ default_statement: if (token->ident == &goto_ident) { stmt->type = STMT_GOTO; token = token->next; - if (token->type == TOKEN_IDENT) { + if (token_type(token) == TOKEN_IDENT) { stmt->goto_label = token; token = token->next; } else - warn(token, "invalid label"); + warn(token->pos, "invalid label"); return expect(token, ';', "at end of statement"); } if (token->ident == &asm_ident || token->ident == &__asm___ident || token->ident == &__asm_ident) { struct expression *expr; stmt->type = STMT_ASM; token = token->next; - if (token->type == TOKEN_IDENT) { + if (token_type(token) == TOKEN_IDENT) { if (token->ident == &__volatile___ident || token->ident == &volatile_ident) token = token->next; } @@ -685,7 +685,7 @@ static struct token *parameter_type_list(struct token *token, struct symbol *fn) { struct symbol_list **list = &fn->arguments; for (;;) { - struct symbol *sym = alloc_symbol(token, SYM_NODE); + struct symbol *sym = alloc_symbol(token->pos, SYM_NODE); if (match_op(token, SPECIAL_ELLIPSIS)) { fn->variadic = 1; @@ -695,7 +695,7 @@ static struct token *parameter_type_list(struct token *token, struct symbol *fn) token = parameter_declaration(token, &sym); /* Special case: (void) */ - if (!*list && !sym->token && sym->ctype.base_type == &void_ctype) + if (!*list && !sym->ident && sym->ctype.base_type == &void_ctype) break; add_symbol(list, sym); if (!match_op(token, ',')) @@ -743,10 +743,10 @@ struct token *initializer(struct token *token, struct ctype *type) struct token *next, *name = NULL; next = token->next; - if (match_op(token, '.') && (next->type == TOKEN_IDENT) && match_op(next->next, '=')) { + if (match_op(token, '.') && (token_type(next) == TOKEN_IDENT) && match_op(next->next, '=')) { name = next; token = next->next->next; - } else if ((token->type == TOKEN_IDENT) && match_op(next, ':')) { + } else if ((token_type(token) == TOKEN_IDENT) && match_op(next, ':')) { name = token; token = next->next; } @@ -763,22 +763,22 @@ static void declare_argument(struct symbol *sym, void *data, int flags) struct symbol *decl = data; if (!sym->ident) { - warn(decl->token, "no identifier for function argument"); + warn(decl->pos, "no identifier for function argument"); return; } - bind_symbol(sym, sym->ident->ident, NS_SYMBOL); + bind_symbol(sym, sym->ident, NS_SYMBOL); } static struct token *external_declaration(struct token *token, struct symbol_list **list) { - struct token *ident = NULL; + struct ident *ident = NULL; struct symbol *decl; struct ctype ctype = { 0, }; struct symbol *base_type; /* Parse declaration-specifiers, if any */ token = declaration_specifiers(token, &ctype); - decl = alloc_symbol(token, SYM_NODE); + decl = alloc_symbol(token->pos, SYM_NODE); decl->ctype = ctype; token = pointer(token, &decl->ctype); token = declarator(token, &decl, &ident); @@ -791,10 +791,10 @@ static struct token *external_declaration(struct token *token, struct symbol_lis /* type define declaration? */ if (ctype.modifiers & MOD_TYPEDEF) { - bind_symbol(decl, ident->ident, NS_TYPEDEF); + bind_symbol(decl, ident, NS_TYPEDEF); } else { add_symbol(list, decl); - bind_symbol(decl, ident->ident, NS_SYMBOL); + bind_symbol(decl, ident, NS_SYMBOL); } base_type = decl->ctype.base_type; @@ -814,20 +814,20 @@ static struct token *external_declaration(struct token *token, struct symbol_lis break; ident = NULL; - decl = alloc_symbol(token, SYM_NODE); + decl = alloc_symbol(token->pos, SYM_NODE); decl->ctype = ctype; token = pointer(token, &decl->ctype); token = declarator(token->next, &decl, &ident); if (!ident) { - warn(token, "expected identifier name in type definition"); + warn(token->pos, "expected identifier name in type definition"); return token; } if (ctype.modifiers & MOD_TYPEDEF) { - bind_symbol(decl, ident->ident, NS_TYPEDEF); + bind_symbol(decl, ident, NS_TYPEDEF); } else { add_symbol(list, decl); - bind_symbol(decl, ident->ident, NS_SYMBOL); + bind_symbol(decl, ident, NS_SYMBOL); } } return expect(token, ';', "at end of declaration"); diff --git a/pre-process.c b/pre-process.c index 5e3f82a..748dabf 100644 --- a/pre-process.c +++ b/pre-process.c @@ -49,14 +49,14 @@ static int match_string_ident(struct ident *ident, const char *str) return !str[ident->len] && !memcmp(str, ident->name, ident->len); } -static struct token *alloc_token(struct token *dup) +static struct token *alloc_token(struct position *pos) { struct token *token = __alloc_token(0); - token->stream = dup->stream; - token->line = dup->line; - token->pos = dup->pos; - token->whitespace = 1; + token->pos.stream = pos->stream; + token->pos.line = pos->line; + token->pos.pos = pos->pos; + token->pos.whitespace = 1; return token; } @@ -72,7 +72,7 @@ static struct token *for_each_ident(struct token *head, struct token *(*action)( if (eof_token(next)) break; - if (next->type == TOKEN_IDENT) + if (token_type(next) == TOKEN_IDENT) next = action(head, next); head = next; @@ -84,9 +84,9 @@ static struct token *is_defined(struct token *head, struct token *token, struct { char *string[] = { "0", "1" }; char *defined = string[lookup_symbol(token->ident, NS_PREPROCESSOR) != NULL]; - struct token *newtoken = alloc_token(token); + struct token *newtoken = alloc_token(&token->pos); - newtoken->type = TOKEN_INTEGER; + token_type(newtoken) = TOKEN_INTEGER; newtoken->integer = defined; newtoken->next = next; head->next = newtoken; @@ -107,7 +107,7 @@ struct token *defined_one_symbol(struct token *head, struct token *next) return next; past = past->next; } - if (token->type == TOKEN_IDENT) + if (token_type(token) == TOKEN_IDENT) return is_defined(head, token, past); } return next; @@ -128,7 +128,7 @@ static void replace_with_string(struct token *token, const char *str) s->length = size; memcpy(s->data, str, size); - token->type = TOKEN_STRING; + token_type(token) = TOKEN_STRING; token->string = s; } @@ -136,7 +136,7 @@ static void replace_with_integer(struct token *token, unsigned int val) { char *buf = __alloc_bytes(10); sprintf(buf, "%d", val); - token->type = TOKEN_INTEGER; + token_type(token) = TOKEN_INTEGER; token->integer = buf; } @@ -149,9 +149,9 @@ struct token *expand_one_symbol(struct token *head, struct token *token) return expand(head, sym); } if (!memcmp(token->ident->name, "__LINE__", 9)) { - replace_with_integer(token, token->line); + replace_with_integer(token, token->pos.line); } else if (!memcmp(token->ident->name, "__FILE__", 9)) { - replace_with_string(token, (input_streams + token->stream)->name); + replace_with_string(token, (input_streams + token->pos.stream)->name); } return token; } @@ -181,12 +181,12 @@ static struct token *find_argument_end(struct token *start) return start; } -static struct token *dup_token(struct token *token, struct token *pos, int newline) +static struct token *dup_token(struct token *token, struct position *pos, int newline) { struct token *alloc = alloc_token(pos); - alloc->type = token->type; - alloc->line = pos->line; - alloc->newline = newline; + token_type(alloc) = token_type(token); + alloc->pos.line = pos->line; + alloc->pos.newline = newline; alloc->integer = token->integer; return alloc; } @@ -199,11 +199,11 @@ static void insert(struct token *token, struct token *prev) static struct token * replace(struct token *token, struct token *prev, struct token *list) { - int newline = token->newline; + int newline = token->pos.newline; prev->next = token->next; while (!eof_token(list) && !match_op(list, SPECIAL_ARG_SEPARATOR)) { - struct token *newtok = dup_token(list, token, newline); + struct token *newtok = dup_token(list, &token->pos, newline); newline = 0; insert(newtok, prev); prev = newtok; @@ -230,13 +230,13 @@ static struct token *stringify(struct token *token, struct token *arg) { const char *s = show_token_sequence(arg); int size = strlen(s)+1; - struct token *newtoken = alloc_token(token); + struct token *newtoken = alloc_token(&token->pos); struct string *string = __alloc_string(size); - newtoken->newline = token->newline; + newtoken->pos.newline = token->pos.newline; memcpy(string->data, s, size); string->length = size; - newtoken->type = TOKEN_STRING; + token_type(newtoken) = TOKEN_STRING; newtoken->string = string; newtoken->next = &eof_token_entry; return newtoken; @@ -255,7 +255,7 @@ static int arg_number(struct token *arglist, struct ident *ident) return -1; } -static struct token empty_arg_token = { .type = TOKEN_EOF }; +static struct token empty_arg_token = { .pos = { .type = TOKEN_EOF } }; static struct token *expand_one_arg(struct token *head, struct token *token, struct token *arglist, struct token *arguments) @@ -299,15 +299,15 @@ static void expand_arguments(struct token *token, struct token *head, if (match_op(next, '#')) { struct token *nextnext = next->next; int nr = arg_number(arglist, nextnext->ident); - if (nextnext != head && nr >= 0 && nextnext->type == TOKEN_IDENT) { + if (nextnext != head && nr >= 0 && token_type(nextnext) == TOKEN_IDENT) { struct token *newtoken = stringify(nextnext, get_argument(nr, arguments)); replace(nextnext, head, newtoken); continue; } - warn(next, "'#' operation is not followed by argument name"); + warn(next->pos, "'#' operation is not followed by argument name"); } - if (next->type == TOKEN_IDENT) + if (token_type(next) == TOKEN_IDENT) next = expand_one_arg(head, next, arglist, arguments); head = next; @@ -339,13 +339,13 @@ static struct token *hashhash(struct token *head, struct token *first, struct to * * See expand_one_arg. */ - if (second->type == TOKEN_EOF) { + if (token_type(second) == TOKEN_EOF) { head->next = second->next; return head; } p = buffer; - switch (first->type) { + switch (token_type(first)) { case TOKEN_INTEGER: len = strlen(first->integer); src = first->integer; @@ -360,7 +360,7 @@ static struct token *hashhash(struct token *head, struct token *first, struct to memcpy(p, src, len); p += len; - switch (second->type) { + switch (token_type(second)) { case TOKEN_INTEGER: len = strlen(second->integer); src = second->integer; @@ -376,10 +376,10 @@ static struct token *hashhash(struct token *head, struct token *first, struct to p += len; *p++ = 0; - newtoken = alloc_token(first); + newtoken = alloc_token(&first->pos); head->next = newtoken; - newtoken->type = first->type; - switch (newtoken->type) { + token_type(newtoken) = token_type(first); + switch (token_type(newtoken)) { case TOKEN_IDENT: newtoken->ident = built_in_ident(buffer); break; @@ -468,7 +468,7 @@ static const char *token_name_sequence(struct token *token, int endop, struct to while (!eof_token(token) && !match_op(token, endop)) { int len; const char *val = token->string->data; - if (token->type != TOKEN_STRING) + if (token_type(token) != TOKEN_STRING) val = show_token(token); len = strlen(val); memcpy(ptr, val, len); @@ -477,7 +477,7 @@ static const char *token_name_sequence(struct token *token, int endop, struct to } *ptr = 0; if (endop && !match_op(token, endop)) - warn(start, "expected '>' at end of filename"); + warn(start->pos, "expected '>' at end of filename"); return buffer; } @@ -501,7 +501,7 @@ static void do_include(struct token *head, struct token *token, const char *file return; } } - warn(token, "unable to open '%s'", filename); + warn(token->pos, "unable to open '%s'", filename); } static int handle_include(struct stream *stream, struct token *head, struct token *token) @@ -534,7 +534,7 @@ static int token_list_different(struct token *list1, struct token *list2) return 0; if (!list1 || !list2) return 1; - if (list1->type != list2->type) + if (token_type(list1) != token_type(list2)) return 1; list1 = list1->next; list2 = list2->next; @@ -549,8 +549,8 @@ static int handle_define(struct stream *stream, struct token *head, struct token struct symbol *sym; struct ident *name; - if (left->type != TOKEN_IDENT) { - warn(head, "expected identifier to 'define'"); + if (token_type(left) != TOKEN_IDENT) { + warn(head->pos, "expected identifier to 'define'"); return 0; } if (false_nesting) @@ -559,7 +559,7 @@ static int handle_define(struct stream *stream, struct token *head, struct token arglist = NULL; expansion = left->next; - if (!expansion->whitespace && match_op(expansion, '(')) { + if (!expansion->pos.whitespace && match_op(expansion, '(')) { arglist = expansion; while (!eof_token(expansion)) { struct token *next = expansion->next; @@ -580,12 +580,12 @@ static int handle_define(struct stream *stream, struct token *head, struct token if (sym) { if (token_list_different(sym->expansion, expansion) || token_list_different(sym->arglist, arglist)) { - warn(left, "preprocessor token redefined"); - warn(sym->token, "this was the original definition"); + warn(left->pos, "preprocessor token redefined"); + warn(sym->pos, "this was the original definition"); } return 1; } - sym = alloc_symbol(left, SYM_NODE); + sym = alloc_symbol(left->pos, SYM_NODE); bind_symbol(sym, name, NS_PREPROCESSOR); sym->expansion = expansion; @@ -598,8 +598,8 @@ static int handle_undef(struct stream *stream, struct token *head, struct token struct token *left = token->next; struct symbol **sym; - if (left->type != TOKEN_IDENT) { - warn(head, "expected identifier to 'undef'"); + if (token_type(left) != TOKEN_IDENT) { + warn(head->pos, "expected identifier to 'undef'"); return 0; } if (false_nesting) @@ -631,10 +631,10 @@ static int preprocessor_if(struct token *token, int true) static int token_defined(struct token *token) { - if (token->type == TOKEN_IDENT) + if (token_type(token) == TOKEN_IDENT) return lookup_symbol(token->ident, NS_PREPROCESSOR) != NULL; - warn(token, "expected identifier for #if[n]def"); + warn(token->pos, "expected identifier for #if[n]def"); return 0; } @@ -648,7 +648,7 @@ static int handle_ifndef(struct stream *stream, struct token *head, struct token struct token *next = token->next; if (stream->constant == -1) { int newconstant = 0; - if (next->type == TOKEN_IDENT) { + if (token_type(next) == TOKEN_IDENT) { if (!stream->protect || stream->protect == next->ident) { newconstant = -2; stream->protect = next->ident; @@ -670,7 +670,7 @@ static int expression_value(struct token *head) expand_list(head); token = constant_expression(head->next, &expr); if (!eof_token(token)) - warn(token, "garbage at end: %s", show_token_sequence(token)); + warn(token->pos, "garbage at end: %s", show_token_sequence(token)); value = get_expression_value(expr); return value != 0; } @@ -703,7 +703,7 @@ static int handle_elif(struct stream * stream, struct token *head, struct token true_nesting--; return 1; } - warn(token, "unmatched '#elif'"); + warn(token->pos, "unmatched '#elif'"); return 1; } @@ -725,7 +725,7 @@ static int handle_else(struct stream *stream, struct token *head, struct token * false_nesting = 1; return 1; } - warn(token, "unmatched #else"); + warn(token->pos, "unmatched #else"); return 1; } @@ -742,7 +742,7 @@ static int handle_endif(struct stream *stream, struct token *head, struct token true_nesting--; return 1; } - warn(token, "unmatched #endif"); + warn(token->pos, "unmatched #endif"); return 1; } @@ -762,7 +762,7 @@ static const char *show_token_sequence(struct token *token) memcpy(ptr, val, len); ptr += len; token = token->next; - whitespace = token->whitespace; + whitespace = token->pos.whitespace; } *ptr = 0; return buffer; @@ -772,7 +772,7 @@ static int handle_warning(struct stream *stream, struct token *head, struct toke { if (false_nesting) return 1; - warn(token, "%s", show_token_sequence(token->next)); + warn(token->pos, "%s", show_token_sequence(token->next)); return 1; } @@ -780,7 +780,7 @@ static int handle_error(struct stream *stream, struct token *head, struct token { if (false_nesting) return 1; - error(token, "%s", show_token_sequence(token->next)); + error(token->pos, "%s", show_token_sequence(token->next)); return 1; } @@ -803,7 +803,7 @@ static void add_path_entry(struct token *token, const char *path) return; } } - warn(token, "too many include path entries"); + warn(token->pos, "too many include path entries"); } static int handle_add_include(struct stream *stream, struct token *head, struct token *token) @@ -812,8 +812,8 @@ static int handle_add_include(struct stream *stream, struct token *head, struct token = token->next; if (eof_token(token)) return 1; - if (token->type != TOKEN_STRING) { - warn(token, "expected path string"); + if (token_type(token) != TOKEN_STRING) { + warn(token->pos, "expected path string"); return 1; } add_path_entry(token, token->string->data); @@ -856,10 +856,10 @@ static void handle_preprocessor_line(struct stream *stream, struct token * head, if (!token) return; - if (token->type == TOKEN_IDENT) + if (token_type(token) == TOKEN_IDENT) if (handle_preprocessor_command(stream, head, token->ident, token)) return; - warn(token, "unrecognized preprocessor line '%s'", show_token_sequence(token)); + warn(token->pos, "unrecognized preprocessor line '%s'", show_token_sequence(token)); } static void preprocessor_line(struct stream *stream, struct token * head) @@ -869,7 +869,7 @@ static void preprocessor_line(struct stream *stream, struct token * head) for (;;) { next = *tp; - if (next->newline) + if (next->pos.newline) break; tp = &next->next; } @@ -882,9 +882,9 @@ static void do_preprocess(struct token *head) { do { struct token *next = head->next; - struct stream *stream = input_streams + next->stream; + struct stream *stream = input_streams + next->pos.stream; - if (next->newline && match_op(next, '#')) { + if (next->pos.newline && match_op(next, '#')) { preprocessor_line(stream, head); continue; } @@ -894,7 +894,7 @@ static void do_preprocess(struct token *head) continue; } - switch (next->type) { + switch (token_type(next)) { case TOKEN_STREAMEND: if (stream->constant == -1 && stream->protect) { stream->constant = 1; @@ -922,12 +922,12 @@ static void do_preprocess(struct token *head) struct token * preprocess(struct token *token) { - struct token header = { 0, }; + struct token header = { }; header.next = token; do_preprocess(&header); if (if_nesting) - warn(unmatched_if, "unmatched preprocessor conditional"); + warn(unmatched_if->pos, "unmatched preprocessor conditional"); // Drop all expressions from pre-processing, they're not used any more. clear_expression_alloc(); diff --git a/show-parse.c b/show-parse.c index 9c84325..82e6bd0 100644 --- a/show-parse.c +++ b/show-parse.c @@ -54,7 +54,7 @@ void show_struct_member(struct symbol *sym, void *data, int flags) { if (flags & ITERATE_FIRST) printf(" { "); - printf("%s:%d:%d at offset %ld", show_token(sym->token), sym->bit_size, sym->alignment, sym->offset); + printf("%s:%d:%d at offset %ld", show_ident(sym->ident), sym->bit_size, sym->alignment, sym->offset); if (sym->fieldwidth) printf("[%d..%d]", sym->bit_offset, sym->bit_offset+sym->fieldwidth-1); if (flags & ITERATE_LAST) @@ -145,15 +145,15 @@ void show_type(struct symbol *sym) return; case SYM_STRUCT: - printf("struct %s", show_token(sym->token)); + printf("struct %s", show_ident(sym->ident)); return; case SYM_UNION: - printf("union %s", show_token(sym->token)); + printf("union %s", show_ident(sym->ident)); return; case SYM_ENUM: - printf("enum %s", show_token(sym->token)); + printf("enum %s", show_ident(sym->ident)); return; case SYM_NODE: { @@ -162,7 +162,7 @@ void show_type(struct symbol *sym) printf("notype"); else show_type(type); - printf(": %s", show_token(sym->token)); + printf(": %s", show_ident(sym->ident)); return; } @@ -172,7 +172,7 @@ void show_type(struct symbol *sym) return; default: - printf("strange type %d '%s' of type ", sym->type, show_token(sym->token)); + printf("strange type %d '%s' of type ", sym->type, show_ident(sym->ident)); show_type(sym->ctype.base_type); return; } @@ -423,11 +423,11 @@ void show_expression(struct expression *expr) break; case EXPR_SYMBOL: if (!expr->symbol) { - warn(expr->token, "undefined symbol '%s'", show_token(expr->token)); + warn(expr->pos, "undefined symbol '%s'", show_ident(expr->symbol_name)); printf("<nosymbol>"); break; } - printf("<%s:", show_token(expr->symbol->token)); + printf("<%s:", show_ident(expr->symbol_name)); show_type(expr->symbol->ctype.base_type); printf(">"); break; @@ -27,11 +27,11 @@ struct symbol *lookup_symbol(struct ident *ident, enum namespace ns) return sym; } -struct symbol *alloc_symbol(struct token *token, int type) +struct symbol *alloc_symbol(struct position pos, int type) { struct symbol *sym = __alloc_symbol(0); sym->type = type; - sym->token = token; + sym->pos = pos; return sym; } @@ -141,7 +141,7 @@ static void examine_bitfield_type(struct symbol *sym) examine_symbol_type(base_type); bit_size = base_type->bit_size; if (sym->fieldwidth > bit_size) { - warn(sym->token, "impossible field-width for this type"); + warn(sym->pos, "impossible field-width for this type"); sym->fieldwidth = bit_size; } alignment = base_type->alignment; @@ -223,7 +223,7 @@ void examine_symbol_type(struct symbol * sym) void bind_symbol(struct symbol *sym, struct ident *ident, enum namespace ns) { if (sym->id_list) { - warn(sym->token, "internal error: symbol type already bound"); + warn(sym->pos, "internal error: symbol type already bound"); return; } sym->namespace = ns; @@ -236,7 +236,7 @@ void bind_symbol(struct symbol *sym, struct ident *ident, enum namespace ns) struct symbol *create_symbol(int stream, const char *name, int type) { struct token *token = built_in_token(stream, name); - struct symbol *sym = alloc_symbol(token, type); + struct symbol *sym = alloc_symbol(token->pos, type); bind_symbol(sym, token->ident, NS_TYPEDEF); return sym; } @@ -52,8 +52,8 @@ struct ctype { struct symbol { enum namespace namespace:8; enum type type:8; - struct token *token; /* Where this symbol was declared */ - struct token *ident; /* What identifier this symbol is associated with */ + struct position pos; /* Where this symbol was declared */ + struct ident *ident; /* What identifier this symbol is associated with */ struct symbol *next_id; /* Next semantic symbol that shares this identifier */ struct symbol **id_list; /* Backpointer to symbol list head */ union { @@ -155,7 +155,7 @@ extern struct ident __asm___ident, extern struct symbol *lookup_symbol(struct ident *, enum namespace); extern void init_symbols(void); -extern struct symbol *alloc_symbol(struct token *, int type); +extern struct symbol *alloc_symbol(struct position, int type); extern void show_type(struct symbol *); extern const char *modifier_string(unsigned long mod); extern void show_symbol(struct symbol *); diff --git a/test-lexing.c b/test-lexing.c index 1cd82c9..9af877e 100644 --- a/test-lexing.c +++ b/test-lexing.c @@ -31,9 +31,9 @@ int main(int argc, char **argv) while (!eof_token(token)) { struct token *next = token->next; char * separator = ""; - if (next->whitespace) + if (next->pos.whitespace) separator = " "; - if (next->newline) + if (next->pos.newline) separator = "\n"; printf("%s%s", show_token(token), separator); token = next; @@ -9,6 +9,7 @@ */ #include <sys/types.h> +#include "lib.h" /* * This describes the pure lexical elements (tokens), with @@ -114,12 +115,7 @@ struct string { * pointers. */ struct token { - unsigned int type:8, - stream:8, - pos:14, - newline:1, - whitespace:1; - unsigned int line; + struct position pos; struct token *next; union { char *integer; @@ -131,6 +127,8 @@ struct token { }; }; +#define token_type(x) ((x)->pos.type) + /* * Last token in the stream - points to itself. * This allows us to not test for NULL pointers @@ -155,12 +153,12 @@ extern struct token *preprocess(struct token *); static inline int match_op(struct token *token, int op) { - return token->type == TOKEN_SPECIAL && token->special == op; + return token->pos.type == TOKEN_SPECIAL && token->special == op; } static inline int match_ident(struct token *token, struct ident *id) { - return token->type == TOKEN_IDENT && token->ident == id; + return token->pos.type == TOKEN_IDENT && token->ident == id; } #endif @@ -26,8 +26,8 @@ static int input_streams_allocated; #define BUFSIZE (8192) typedef struct { - int fd, stream, line, pos, offset, size; - unsigned int newline:1, whitespace:1; + int fd, offset, size; + struct position pos; struct token **tokenlist; struct token *token; unsigned char *buffer; @@ -49,6 +49,8 @@ const char *show_special(int val) const char *show_ident(const struct ident *ident) { static char buffer[256]; + if (!ident) + return "<noident>"; sprintf(buffer, "%.*s", ident->len, ident->name); return buffer; } @@ -82,7 +84,7 @@ const char *show_token(const struct token *token) if (!token) return "<no token>"; - switch (token->type) { + switch (token_type(token)) { case TOKEN_ERROR: return "syntax error"; @@ -138,11 +140,11 @@ const char *show_token(const struct token *token) } case TOKEN_STREAMBEGIN: - sprintf(buffer, "<beginning of '%s'>", (input_streams + token->stream)->name); + sprintf(buffer, "<beginning of '%s'>", (input_streams + token->pos.stream)->name); return buffer; case TOKEN_STREAMEND: - sprintf(buffer, "<end of '%s'>", (input_streams + token->stream)->name); + sprintf(buffer, "<end of '%s'>", (input_streams + token->pos.stream)->name); return buffer; default: @@ -189,11 +191,7 @@ int init_stream(const char *name, int fd) static struct token * alloc_token(stream_t *stream) { struct token *token = __alloc_token(0); - token->line = stream->line; token->pos = stream->pos; - token->stream = stream->stream; - token->newline = stream->newline; - token->whitespace = stream->whitespace; return token; } @@ -213,11 +211,11 @@ static int nextchar(stream_t *stream) } c = stream->buffer[offset]; stream->offset = offset + 1; - stream->pos++; + stream->pos.pos++; if (c == '\n') { - stream->line++; - stream->newline = 1; - stream->pos = 0; + stream->pos.line++; + stream->pos.newline = 1; + stream->pos.pos = 0; } return c; } @@ -229,11 +227,11 @@ static void mark_eof(stream_t *stream, struct token *end_token) struct token *end; end = alloc_token(stream); - end->type = TOKEN_STREAMEND; - end->newline = 1; + token_type(end) = TOKEN_STREAMEND; + end->pos.newline = 1; eof_token_entry.next = &eof_token_entry; - eof_token_entry.newline = 1; + eof_token_entry.pos.newline = 1; if (!end_token) end_token = &eof_token_entry; @@ -254,8 +252,8 @@ static void add_token(stream_t *stream) static void drop_token(stream_t *stream) { - stream->newline |= stream->token->newline; - stream->whitespace |= stream->token->whitespace; + stream->pos.newline |= stream->token->pos.newline; + stream->pos.whitespace |= stream->token->pos.whitespace; stream->token = NULL; } @@ -288,7 +286,7 @@ static int do_integer(char *buffer, int len, int next, stream_t *stream) buffer[len++] = '\0'; buf = __alloc_bytes(len); memcpy(buf, buffer, len); - token->type = TOKEN_INTEGER; + token_type(token) = TOKEN_INTEGER; token->integer = buf; add_token(stream); return next; @@ -329,7 +327,7 @@ static int escapechar(int first, int type, stream_t *stream, int *valp) value = first; if (first == '\n') - warn(stream->token, "Newline in string or character constant"); + warn(stream->pos, "Newline in string or character constant"); if (first == '\\' && next != EOF) { value = next; @@ -375,7 +373,7 @@ static int escapechar(int first, int type, stream_t *stream, int *valp) } /* Fallthrough */ default: - warn(stream->token, "Unknown escape '%c'", value); + warn(stream->pos, "Unknown escape '%c'", value); } } /* Mark it as escaped */ @@ -392,13 +390,13 @@ static int get_char_token(int next, stream_t *stream) next = escapechar(next, '\'', stream, &value); if (value == '\'' || next != '\'') { - warn(stream->token, "Bad character constant"); + warn(stream->pos, "Bad character constant"); drop_token(stream); return next; } token = stream->token; - token->type = TOKEN_CHAR; + token_type(token) = TOKEN_CHAR; token->character = value & 0xff; add_token(stream); @@ -418,7 +416,7 @@ static int get_string_token(int next, stream_t *stream) if (val == '"') break; if (next == EOF) { - warn(stream->token, "Enf of file in middle of string"); + warn(stream->pos, "Enf of file in middle of string"); return next; } if (len < sizeof(buffer)) { @@ -429,7 +427,7 @@ static int get_string_token(int next, stream_t *stream) } if (len > 256) - warn(stream->token, "String too long"); + warn(stream->pos, "String too long"); string = __alloc_string(len+1); memcpy(string->data, buffer, len); @@ -438,7 +436,7 @@ static int get_string_token(int next, stream_t *stream) /* Pass it on.. */ token = stream->token; - token->type = TOKEN_STRING; + token_type(token) = TOKEN_STRING; token->string = string; add_token(stream); @@ -466,7 +464,7 @@ static int drop_stream_comment(stream_t *stream) for (;;) { int curr = next; if (curr == EOF) { - warn(stream->token, "End of file in the middle of a comment"); + warn(stream->pos, "End of file in the middle of a comment"); return curr; } next = nextchar(stream); @@ -523,7 +521,7 @@ static int get_one_special(int c, stream_t *stream) /* Pass it on.. */ token = stream->token; - token->type = TOKEN_SPECIAL; + token_type(token) = TOKEN_SPECIAL; token->special = value; add_token(stream); return next; @@ -632,8 +630,8 @@ struct token *built_in_token(int stream, const char *name) struct token *token; token = __alloc_token(0); - token->stream = stream; - token->type = TOKEN_IDENT; + token->pos.stream = stream; + token_type(token) = TOKEN_IDENT; token->ident = built_in_ident(name); return token; } @@ -671,7 +669,7 @@ static int get_one_identifier(int c, stream_t *stream) /* Pass it on.. */ token = stream->token; - token->type = TOKEN_IDENT; + token_type(token) = TOKEN_IDENT; token->ident = ident; add_token(stream); return next; @@ -696,19 +694,20 @@ static struct token *setup_stream(stream_t *stream, int idx, int fd, { struct token *begin; - stream->stream = idx; + stream->pos.stream = idx; + stream->pos.line = 1; + stream->pos.newline = 1; + stream->pos.whitespace = 0; + stream->pos.pos = 0; + stream->token = NULL; - stream->line = 1; - stream->newline = 1; - stream->whitespace = 0; - stream->pos = 0; stream->fd = fd; stream->offset = 0; stream->size = buf_size; stream->buffer = buf; begin = alloc_token(stream); - begin->type = TOKEN_STREAMBEGIN; + token_type(begin) = TOKEN_STREAMBEGIN; stream->tokenlist = &begin->next; return begin; } @@ -719,21 +718,19 @@ static void tokenize_stream(stream_t *stream, struct token *endtoken) while (c != EOF) { if (c == '\\') { c = nextchar(stream); - stream->newline = 0; - stream->whitespace = 1; + stream->pos.newline = 0; + stream->pos.whitespace = 1; continue; } if (!isspace(c)) { struct token *token = alloc_token(stream); - token->newline = stream->newline; - token->whitespace = stream->whitespace; - stream->newline = 0; - stream->whitespace = 0; stream->token = token; + stream->pos.newline = 0; + stream->pos.whitespace = 0; c = get_one_token(c, stream); continue; } - stream->whitespace = 1; + stream->pos.whitespace = 1; c = nextchar(stream); } mark_eof(stream, endtoken); |