Get Gentoo!
gentoo.org sites
gentoo.org Wiki Bugs Forums Packages
Planet Archives Sources
Infra Status
aboutsummaryrefslogtreecommitdiff
diff options
context:
space:
mode:
Diffstat
-rw-r--r--evaluate.c63
-rw-r--r--expression.c55
-rw-r--r--expression.h16
-rw-r--r--lib.c26
-rw-r--r--lib.h13
-rw-r--r--parse.c100
-rw-r--r--pre-process.c130
-rw-r--r--show-parse.c16
-rw-r--r--symbol.c10
-rw-r--r--symbol.h6
-rw-r--r--test-lexing.c4
-rw-r--r--token.h14
-rw-r--r--tokenize.c85
13 files changed, 273 insertions, 265 deletions
diff --git a/evaluate.c b/evaluate.c
index 287556b..93b8e5c 100644
--- a/evaluate.c
+++ b/evaluate.c
@@ -27,7 +27,7 @@ static int evaluate_symbol(struct expression *expr)
struct symbol *base_type;
if (!sym) {
- warn(expr->token, "undefined identifier '%s'", show_token(expr->token));
+ warn(expr->pos, "undefined identifier '%s'", show_ident(expr->symbol_name));
return 0;
}
examine_symbol_type(sym);
@@ -112,7 +112,7 @@ static int get_int_value(struct expression *expr, const char *str)
if (BITS_IN_LONG == BITS_IN_INT)
modifiers = MOD_LONG | MOD_UNSIGNED;
}
- warn(expr->token, "value is so big it is%s%s%s",
+ warn(expr->pos, "value is so big it is%s%s%s",
(modifiers & MOD_UNSIGNED) ? " unsigned":"",
(modifiers & MOD_LONG) ? " long":"",
(modifiers & MOD_LONGLONG) ? " long":"");
@@ -128,7 +128,7 @@ static int evaluate_constant(struct expression *expr)
{
struct token *token = expr->token;
- switch (token->type) {
+ switch (token_type(token)) {
case TOKEN_INTEGER:
return get_int_value(expr, token->integer);
@@ -137,11 +137,13 @@ static int evaluate_constant(struct expression *expr)
expr->ctype = &int_ctype;
expr->value = (char) token->character;
return 1;
+
case TOKEN_STRING:
expr->ctype = &string_ctype;
return 1;
+
default:
- warn(token, "non-typed expression");
+ warn(expr->pos, "non-typed expression");
}
return 0;
}
@@ -212,7 +214,7 @@ static int cast_value(struct expression *expr, struct symbol *newtype,
static struct expression * cast_to(struct expression *old, struct symbol *type)
{
- struct expression *expr = alloc_expression(old->token, EXPR_CAST);
+ struct expression *expr = alloc_expression(old->pos, EXPR_CAST);
expr->ctype = type;
expr->cast_type = type;
expr->cast_expression = old;
@@ -233,7 +235,7 @@ static int is_int_type(struct symbol *type)
static int bad_expr_type(struct expression *expr)
{
- warn(expr->token, "incompatible types for operation");
+ warn(expr->pos, "incompatible types for operation");
return 0;
}
@@ -274,8 +276,10 @@ static int evaluate_int_binop(struct expression *expr)
static struct symbol *degenerate(struct expression *expr, struct symbol *ctype)
{
if (ctype->type == SYM_ARRAY) {
- struct symbol *sym = alloc_symbol(expr->token, SYM_PTR);
+ struct symbol *sym = alloc_symbol(expr->pos, SYM_PTR);
sym->ctype = ctype->ctype;
+ sym->bit_size = BITS_IN_POINTER;
+ sym->alignment = POINTER_ALIGNMENT;
ctype = sym;
}
return ctype;
@@ -298,8 +302,8 @@ static int evaluate_ptr_add(struct expression *expr, struct expression *ptr, str
expr->ctype = degenerate(expr, ptr_type);
if (ctype->bit_size > BITS_IN_CHAR) {
struct expression *add = expr;
- struct expression *mul = alloc_expression(expr->token, EXPR_BINOP);
- struct expression *val = alloc_expression(expr->token, EXPR_VALUE);
+ struct expression *mul = alloc_expression(expr->pos, EXPR_BINOP);
+ struct expression *val = alloc_expression(expr->pos, EXPR_VALUE);
val->ctype = size_t_ctype;
val->value = ctype->bit_size >> 3;
@@ -310,7 +314,6 @@ static int evaluate_ptr_add(struct expression *expr, struct expression *ptr, str
mul->right = val;
/* Leave 'add->op' as 'expr->op' - either '+' or '-' */
- add->ctype = ptr_type;
add->left = ptr;
add->right = mul;
}
@@ -360,16 +363,16 @@ static int evaluate_ptr_sub(struct expression *expr, struct expression *l, struc
ctype = same_ptr_types(ltype, rtype);
if (!ctype) {
- warn(expr->token, "subtraction of different types can't work");
+ warn(expr->pos, "subtraction of different types can't work");
return 0;
}
examine_symbol_type(ctype);
expr->ctype = ssize_t_ctype;
if (ctype->bit_size > BITS_IN_CHAR) {
- struct expression *sub = alloc_expression(expr->token, EXPR_BINOP);
+ struct expression *sub = alloc_expression(expr->pos, EXPR_BINOP);
struct expression *div = expr;
- struct expression *val = alloc_expression(expr->token, EXPR_VALUE);
+ struct expression *val = alloc_expression(expr->pos, EXPR_VALUE);
val->ctype = size_t_ctype;
val->value = ctype->bit_size >> 3;
@@ -489,19 +492,19 @@ static int evaluate_preop(struct expression *expr)
case '*':
if (ctype->type != SYM_PTR && ctype->type != SYM_ARRAY) {
- warn(expr->token, "cannot derefence this type");
+ warn(expr->pos, "cannot derefence this type");
return 0;
}
examine_symbol_type(expr->ctype);
expr->ctype = ctype->ctype.base_type;
if (!expr->ctype) {
- warn(expr->token, "undefined type");
+ warn(expr->pos, "undefined type");
return 0;
}
return 1;
case '&': {
- struct symbol *symbol = alloc_symbol(expr->token, SYM_PTR);
+ struct symbol *symbol = alloc_symbol(expr->pos, SYM_PTR);
symbol->ctype.base_type = ctype;
symbol->bit_size = BITS_IN_POINTER;
symbol->alignment = POINTER_ALIGNMENT;
@@ -538,7 +541,7 @@ struct symbol *find_identifier(struct ident *ident, struct symbol_list *_list, i
for (i = 0; i < list->nr; i++) {
struct symbol *sym = (struct symbol *) list->list[i];
if (sym->ident) {
- if (sym->ident->ident != ident)
+ if (sym->ident != ident)
continue;
*offset = sym->offset;
return sym;
@@ -575,29 +578,29 @@ static int evaluate_dereference(struct expression *expr)
if (expr->op == SPECIAL_DEREFERENCE) {
/* Arrays will degenerate into pointers for '->' */
if (ctype->type != SYM_PTR && ctype->type != SYM_ARRAY) {
- warn(expr->token, "expected a pointer to a struct/union");
+ warn(expr->pos, "expected a pointer to a struct/union");
return 0;
}
ctype = ctype->ctype.base_type;
}
if (!ctype || (ctype->type != SYM_STRUCT && ctype->type != SYM_UNION)) {
- warn(expr->token, "expected structure or union");
+ warn(expr->pos, "expected structure or union");
return 0;
}
offset = 0;
member = find_identifier(token->ident, ctype->symbol_list, &offset);
if (!member) {
- warn(expr->token, "no such struct/union member");
+ warn(expr->pos, "no such struct/union member");
return 0;
}
add = deref;
if (offset != 0) {
- add = alloc_expression(expr->token, EXPR_BINOP);
+ add = alloc_expression(expr->pos, EXPR_BINOP);
add->op = '+';
add->ctype = &ptr_ctype;
add->left = deref;
- add->right = alloc_expression(expr->token, EXPR_VALUE);
+ add->right = alloc_expression(expr->pos, EXPR_VALUE);
add->right->ctype = &int_ctype;
add->right->value = offset;
}
@@ -632,7 +635,7 @@ static int evaluate_sizeof(struct expression *expr)
size = expr->cast_expression->ctype->bit_size;
}
if (size & 7) {
- warn(expr->token, "cannot size expression");
+ warn(expr->pos, "cannot size expression");
return 0;
}
expr->type = EXPR_VALUE;
@@ -694,15 +697,15 @@ static int evaluate_call(struct expression *expr)
if (ctype->type == SYM_PTR || ctype->type == SYM_ARRAY)
ctype = ctype->ctype.base_type;
if (ctype->type != SYM_FN) {
- warn(expr->token, "not a function");
+ warn(expr->pos, "not a function");
return 0;
}
args = expression_list_size(expr->args);
fnargs = symbol_list_size(ctype->arguments);
if (args < fnargs)
- warn(expr->token, "not enough arguments for function");
+ warn(expr->pos, "not enough arguments for function");
if (args > fnargs && !ctype->variadic)
- warn(expr->token, "too many arguments for function");
+ warn(expr->pos, "too many arguments for function");
expr->ctype = ctype->ctype.base_type;
return 1;
}
@@ -774,12 +777,12 @@ long long get_expression_value(struct expression *expr)
if (expr->cast_type) {
examine_symbol_type(expr->cast_type);
if (expr->cast_type->bit_size & 7) {
- warn(expr->token, "type has no size");
+ warn(expr->pos, "type has no size");
return 0;
}
return expr->cast_type->bit_size >> 3;
}
- warn(expr->token, "expression sizes not yet supported");
+ warn(expr->pos, "expression sizes not yet supported");
return 0;
case EXPR_CONSTANT:
evaluate_constant(expr);
@@ -789,7 +792,7 @@ long long get_expression_value(struct expression *expr)
case EXPR_SYMBOL: {
struct symbol *sym = expr->symbol;
if (!sym || !sym->ctype.base_type || sym->ctype.base_type->type != SYM_ENUM) {
- warn(expr->token, "undefined identifier in constant expression");
+ warn(expr->pos, "undefined identifier '%s' in constant expression", expr->symbol_name);
return 0;
}
return sym->value;
@@ -836,6 +839,6 @@ long long get_expression_value(struct expression *expr)
default:
break;
}
- error(expr->token, "bad constant expression");
+ error(expr->pos, "bad constant expression");
return 0;
}
diff --git a/expression.c b/expression.c
index 62f196a..2062f88 100644
--- a/expression.c
+++ b/expression.c
@@ -40,7 +40,7 @@ struct token *parens_expression(struct token *token, struct expression **expr, c
{
token = expect(token, '(', where);
if (match_op(token, '{')) {
- struct expression *e = alloc_expression(token, EXPR_STATEMENT);
+ struct expression *e = alloc_expression(token->pos, EXPR_STATEMENT);
struct statement *stmt = alloc_statement(token, STMT_COMPOUND);
*expr = e;
e->statement = stmt;
@@ -57,31 +57,34 @@ struct token *primary_expression(struct token *token, struct expression **tree)
{
struct expression *expr = NULL;
- switch (token->type) {
+ switch (token_type(token)) {
case TOKEN_INTEGER:
case TOKEN_FP:
case TOKEN_CHAR:
- expr = alloc_expression(token, EXPR_CONSTANT);
+ expr = alloc_expression(token->pos, EXPR_CONSTANT);
+ expr->token = token;
token = token->next;
break;
case TOKEN_IDENT: {
- expr = alloc_expression(token, EXPR_SYMBOL);
+ expr = alloc_expression(token->pos, EXPR_SYMBOL);
+ expr->symbol_name = token->ident;
expr->symbol = lookup_symbol(token->ident, NS_SYMBOL);
token = token->next;
break;
}
case TOKEN_STRING:
- expr = alloc_expression(token, EXPR_CONSTANT);
+ expr = alloc_expression(token->pos, EXPR_CONSTANT);
+ expr->token = token;
do {
token = token->next;
- } while (token->type == TOKEN_STRING);
+ } while (token_type(token) == TOKEN_STRING);
break;
case TOKEN_SPECIAL:
if (token->special == '(') {
- expr = alloc_expression(token, EXPR_PREOP);
+ expr = alloc_expression(token->pos, EXPR_PREOP);
expr->op = '(';
token = parens_expression(token, &expr->unop, "in expression");
break;
@@ -113,11 +116,11 @@ static struct token *postfix_expression(struct token *token, struct expression *
struct expression *expr = NULL;
token = primary_expression(token, &expr);
- while (expr && token->type == TOKEN_SPECIAL) {
+ while (expr && token_type(token) == TOKEN_SPECIAL) {
switch (token->special) {
case '[': { /* Array dereference */
- struct expression *deref = alloc_expression(token, EXPR_PREOP);
- struct expression *add = alloc_expression(token, EXPR_BINOP);
+ struct expression *deref = alloc_expression(token->pos, EXPR_PREOP);
+ struct expression *add = alloc_expression(token->pos, EXPR_BINOP);
deref->op = '*';
deref->unop = add;
@@ -131,7 +134,7 @@ static struct token *postfix_expression(struct token *token, struct expression *
}
case SPECIAL_INCREMENT: /* Post-increment */
case SPECIAL_DECREMENT: { /* Post-decrement */
- struct expression *post = alloc_expression(token, EXPR_POSTOP);
+ struct expression *post = alloc_expression(token->pos, EXPR_POSTOP);
post->op = token->special;
post->unop = expr;
expr = post;
@@ -140,12 +143,12 @@ static struct token *postfix_expression(struct token *token, struct expression *
}
case '.': /* Structure member dereference */
case SPECIAL_DEREFERENCE: { /* Structure pointer member dereference */
- struct expression *deref = alloc_expression(token, EXPR_DEREF);
+ struct expression *deref = alloc_expression(token->pos, EXPR_DEREF);
deref->op = token->special;
deref->deref = expr;
token = token->next;
- if (token->type != TOKEN_IDENT) {
- warn(token, "Expected member name");
+ if (token_type(token) != TOKEN_IDENT) {
+ warn(token->pos, "Expected member name");
break;
}
deref->member = token;
@@ -155,7 +158,7 @@ static struct token *postfix_expression(struct token *token, struct expression *
}
case '(': { /* Function call */
- struct expression *call = alloc_expression(token, EXPR_CALL);
+ struct expression *call = alloc_expression(token->pos, EXPR_CALL);
call->op = '(';
call->fn = expr;
token = expression_list(token->next, &call->args);
@@ -176,10 +179,10 @@ static struct token *postfix_expression(struct token *token, struct expression *
static struct token *cast_expression(struct token *token, struct expression **tree);
static struct token *unary_expression(struct token *token, struct expression **tree)
{
- if (token->type == TOKEN_IDENT &&
+ if (token_type(token) == TOKEN_IDENT &&
(token->ident == &sizeof_ident ||
token->ident == &__alignof___ident)) {
- struct expression *sizeof_ex = alloc_expression(token, EXPR_SIZEOF);
+ struct expression *sizeof_ex = alloc_expression(token->pos, EXPR_SIZEOF);
*tree = sizeof_ex;
tree = &sizeof_ex->unop;
token = token->next;
@@ -189,11 +192,11 @@ static struct token *unary_expression(struct token *token, struct expression **t
return expect(token, ')', "at end of sizeof type-name");
}
- if (token->type == TOKEN_SPECIAL) {
+ if (token_type(token) == TOKEN_SPECIAL) {
if (match_oplist(token->special,
SPECIAL_INCREMENT, SPECIAL_DECREMENT,
'&', '*', '+', '-', '~', '!', 0)) {
- struct expression *unary = alloc_expression(token, EXPR_PREOP);
+ struct expression *unary = alloc_expression(token->pos, EXPR_PREOP);
unary->op = token->special;
*tree = unary;
return cast_expression(token->next, &unary->unop);
@@ -213,7 +216,7 @@ static struct token *cast_expression(struct token *token, struct expression **tr
if (match_op(token, '(')) {
struct token *next = token->next;
if (lookup_type(next)) {
- struct expression *cast = alloc_expression(next, EXPR_CAST);
+ struct expression *cast = alloc_expression(next->pos, EXPR_CAST);
struct symbol *sym;
token = typename(next, &sym);
@@ -237,7 +240,7 @@ static struct token *lr_binop_expression(struct token *token, struct expression
struct token * next = inner(token, &left);
if (left) {
- while (next->type == TOKEN_SPECIAL) {
+ while (token_type(next) == TOKEN_SPECIAL) {
struct expression *top, *right = NULL;
int op = next->special;
va_list args;
@@ -251,10 +254,10 @@ static struct token *lr_binop_expression(struct token *token, struct expression
break;
}
va_end(args);
- top = alloc_expression(next, type);
+ top = alloc_expression(next->pos, type);
next = inner(next->next, &right);
if (!right) {
- warn(next, "No right hand side of '%s'-expression", show_special(op));
+ warn(next->pos, "No right hand side of '%s'-expression", show_special(op));
break;
}
top->op = op;
@@ -322,7 +325,7 @@ struct token *conditional_expression(struct token *token, struct expression **tr
{
token = logical_or_expression(token, tree);
if (match_op(token, '?')) {
- struct expression *expr = alloc_expression(token, EXPR_CONDITIONAL);
+ struct expression *expr = alloc_expression(token->pos, EXPR_CONDITIONAL);
expr->op = token->special;
expr->left = *tree;
*tree = expr;
@@ -336,7 +339,7 @@ struct token *conditional_expression(struct token *token, struct expression **tr
struct token *assignment_expression(struct token *token, struct expression **tree)
{
token = conditional_expression(token, tree);
- if (token->type == TOKEN_SPECIAL) {
+ if (token_type(token) == TOKEN_SPECIAL) {
static const int assignments[] = {
'=', SPECIAL_ADD_ASSIGN, SPECIAL_MINUS_ASSIGN,
SPECIAL_TIMES_ASSIGN, SPECIAL_DIV_ASSIGN,
@@ -346,7 +349,7 @@ struct token *assignment_expression(struct token *token, struct expression **tre
int i, op = token->special;
for (i = 0; i < sizeof(assignments)/sizeof(int); i++)
if (assignments[i] == op) {
- struct expression * expr = alloc_expression(token, EXPR_ASSIGNMENT);
+ struct expression * expr = alloc_expression(token->pos, EXPR_ASSIGNMENT);
expr->left = *tree;
expr->op = op;
*tree = expr;
diff --git a/expression.h b/expression.h
index bf201ba..1897153 100644
--- a/expression.h
+++ b/expression.h
@@ -32,9 +32,12 @@ enum expression_type {
struct expression {
enum expression_type type;
int op;
- struct token *token;
+ struct position pos;
struct symbol *ctype;
union {
+ // EXPR_CONSTANT
+ struct token *token;
+
// EXPR_VALUE
unsigned long long value;
@@ -42,7 +45,10 @@ struct expression {
struct expression *unop;
// EXPR_SYMBOL
- struct symbol *symbol;
+ struct symbol_arg {
+ struct symbol *symbol;
+ struct ident *symbol_name;
+ };
// EXPR_STATEMENT
struct statement *statement;
@@ -90,11 +96,11 @@ struct token *assignment_expression(struct token *token, struct expression **tre
extern int evaluate_expression(struct expression *);
-static inline struct expression *alloc_expression(struct token *token, int type)
+static inline struct expression *alloc_expression(struct position pos, int type)
{
struct expression *expr = __alloc_expression(0);
expr->type = type;
- expr->token = token;
+ expr->pos = pos;
return expr;
}
@@ -103,7 +109,7 @@ struct token *typename(struct token *, struct symbol **);
static inline int lookup_type(struct token *token)
{
- if (token->type == TOKEN_IDENT)
+ if (token->pos.type == TOKEN_IDENT)
return lookup_symbol(token->ident, NS_TYPEDEF) != NULL;
return 0;
}
diff --git a/lib.c b/lib.c
index 8d5312e..1276352 100644
--- a/lib.c
+++ b/lib.c
@@ -28,8 +28,8 @@ struct token *expect(struct token *token, int op, const char *where)
static struct token bad_token;
if (token != &bad_token) {
bad_token.next = token;
- warn(token, "Expected %s %s", show_special(op), where);
- warn(token, "got %s", show_token(token));
+ warn(token->pos, "Expected %s %s", show_special(op), where);
+ warn(token->pos, "got %s", show_token(token));
}
if (op == ';')
return skip_to(token, op);
@@ -223,27 +223,19 @@ void add_ptr_list(struct ptr_list **listp, void *ptr)
list->nr = nr;
}
-static void do_warn(const char *type, struct token *token, const char * fmt, va_list args)
+static void do_warn(const char *type, struct position pos, const char * fmt, va_list args)
{
static char buffer[512];
const char *name;
- int pos,line;
vsprintf(buffer, fmt, args);
- name = "EOF";
- pos = 0;
- line = 0;
- if (token) {
- name = input_streams[token->stream].name;
- pos = token->pos;
- line = token->line;
- }
+ name = input_streams[pos.stream].name;
fprintf(stderr, "%s: %s:%d:%d: %s\n",
- type, name, line, pos, buffer);
+ type, name, pos.line, pos.pos, buffer);
}
-void warn(struct token *token, const char * fmt, ...)
+void warn(struct position pos, const char * fmt, ...)
{
static int warnings = 0;
@@ -257,16 +249,16 @@ void warn(struct token *token, const char * fmt, ...)
va_list args;
va_start(args, fmt);
- do_warn("warning", token, fmt, args);
+ do_warn("warning", pos, fmt, args);
va_end(args);
warnings++;
}
-void error(struct token *token, const char * fmt, ...)
+void error(struct position pos, const char * fmt, ...)
{
va_list args;
va_start(args, fmt);
- do_warn("error", token, fmt, args);
+ do_warn("error", pos, fmt, args);
va_end(args);
exit(1);
}
diff --git a/lib.h b/lib.h
index 34c51b1..12103ee 100644
--- a/lib.h
+++ b/lib.h
@@ -8,6 +8,15 @@
extern unsigned int hexval(unsigned int c);
+struct position {
+ unsigned int type:6,
+ stream:10,
+ pos:14,
+ newline:1,
+ whitespace:1;
+ unsigned int line;
+};
+
struct ident;
struct token;
struct symbol;
@@ -19,8 +28,8 @@ struct expression_list;
struct token *skip_to(struct token *, int);
struct token *expect(struct token *, int, const char *);
-extern void warn(struct token *, const char *, ...);
-extern void error(struct token *, const char *, ...);
+extern void warn(struct position, const char *, ...);
+extern void error(struct position, const char *, ...);
#define __DECLARE_ALLOCATOR(type, x) \
extern type *__alloc_##x(int); \
diff --git a/parse.c b/parse.c
index 053771c..bd5c441 100644
--- a/parse.c
+++ b/parse.c
@@ -30,9 +30,9 @@ struct statement *alloc_statement(struct token * token, int type)
static struct token *struct_declaration_list(struct token *token, struct symbol_list **list);
-static struct symbol * indirect(struct token *token, struct ctype *ctype, int type)
+static struct symbol * indirect(struct position pos, struct ctype *ctype, int type)
{
- struct symbol *sym = alloc_symbol(token, type);
+ struct symbol *sym = alloc_symbol(pos, type);
sym->ctype.base_type = ctype->base_type;
sym->ctype.modifiers = ctype->modifiers & ~MOD_STORAGE;
@@ -46,7 +46,8 @@ static struct symbol *lookup_or_create_symbol(enum namespace ns, enum type type,
{
struct symbol *sym = lookup_symbol(token->ident, ns);
if (!sym) {
- sym = alloc_symbol(token, type);
+ sym = alloc_symbol(token->pos, type);
+ sym->ident = token->ident;
bind_symbol(sym, token->ident, ns);
}
return sym;
@@ -59,7 +60,7 @@ struct token *struct_union_enum_specifier(enum namespace ns, enum type type,
struct symbol *sym;
ctype->modifiers = 0;
- if (token->type == TOKEN_IDENT) {
+ if (token_type(token) == TOKEN_IDENT) {
sym = lookup_or_create_symbol(ns, type, token);
token = token->next;
ctype->base_type = sym;
@@ -72,12 +73,12 @@ struct token *struct_union_enum_specifier(enum namespace ns, enum type type,
// private struct/union/enum type
if (!match_op(token, '{')) {
- warn(token, "expected declaration");
+ warn(token->pos, "expected declaration");
ctype->base_type = &bad_type;
return token;
}
- sym = alloc_symbol(token, type);
+ sym = alloc_symbol(token->pos, type);
token = parse(token->next, sym);
ctype->base_type = sym;
return expect(token, '}', "at end of specifier");
@@ -96,11 +97,11 @@ struct token *struct_or_union_specifier(enum type type, struct token *token, str
static struct token *parse_enum_declaration(struct token *token, struct symbol *parent)
{
int nextval = 0;
- while (token->type == TOKEN_IDENT) {
+ while (token_type(token) == TOKEN_IDENT) {
struct token *next = token->next;
struct symbol *sym;
- sym = alloc_symbol(token, SYM_NODE);
+ sym = alloc_symbol(token->pos, SYM_NODE);
bind_symbol(sym, token->ident, NS_SYMBOL);
sym->ctype.base_type = parent;
@@ -131,7 +132,7 @@ struct token *typeof_specifier(struct token *token, struct ctype *ctype)
struct expression *expr;
if (!match_op(token, '(')) {
- warn(token, "expected '(' after typeof");
+ warn(token->pos, "expected '(' after typeof");
return token;
}
if (lookup_type(token->next)) {
@@ -183,7 +184,7 @@ static struct token *type_qualifiers(struct token *next, struct ctype *ctype)
unsigned long mod;
next = token->next;
- if (token->type != TOKEN_IDENT)
+ if (token_type(token) != TOKEN_IDENT)
break;
s = lookup_symbol(token->ident, NS_TYPEDEF);
if (!s)
@@ -247,7 +248,7 @@ static struct token *declaration_specifiers(struct token *next, struct ctype *ct
unsigned long mod;
next = token->next;
- if (token->type != TOKEN_IDENT)
+ if (token_type(token) != TOKEN_IDENT)
break;
ident = token->ident;
@@ -274,7 +275,7 @@ static struct token *declaration_specifiers(struct token *next, struct ctype *ct
if (type) {
if (type != ctype->base_type) {
if (ctype->base_type) {
- warn(token, "Strange mix of types");
+ warn(token->pos, "Strange mix of types");
continue;
}
ctype->base_type = type;
@@ -291,7 +292,7 @@ static struct token *declaration_specifiers(struct token *next, struct ctype *ct
}
dup = (mod & old) | (extra & old) | (extra & mod);
if (dup)
- warn(token, "Just how %s do you want this type to be?",
+ warn(token->pos, "Just how %s do you want this type to be?",
modifier_string(dup));
ctype->modifiers = old | mod | extra;
}
@@ -327,14 +328,14 @@ static struct token *abstract_array_declarator(struct token *token, struct symbo
}
static struct token *parameter_type_list(struct token *, struct symbol *);
-static struct token *declarator(struct token *token, struct symbol **tree, struct token **p);
+static struct token *declarator(struct token *token, struct symbol **tree, struct ident **p);
-static struct token *direct_declarator(struct token *token, struct symbol **tree, struct token **p)
+static struct token *direct_declarator(struct token *token, struct symbol **tree, struct ident **p)
{
struct ctype *ctype = &(*tree)->ctype;
- if (p && token->type == TOKEN_IDENT) {
- *p = token;
+ if (p && token_type(token) == TOKEN_IDENT) {
+ *p = token->ident;
token = token->next;
}
@@ -344,7 +345,7 @@ static struct token *direct_declarator(struct token *token, struct symbol **tree
token = attribute_specifier(token->next, &ctype);
continue;
}
- if (token->type != TOKEN_SPECIAL)
+ if (token_type(token) != TOKEN_SPECIAL)
return token;
/*
@@ -369,13 +370,13 @@ static struct token *direct_declarator(struct token *token, struct symbol **tree
continue;
}
- sym = indirect(token, ctype, SYM_FN);
+ sym = indirect(token->pos, ctype, SYM_FN);
token = parameter_type_list(next, sym);
token = expect(token, ')', "in function declarator");
continue;
}
if (token->special == '[') {
- struct symbol *array = indirect(token, ctype, SYM_ARRAY);
+ struct symbol *array = indirect(token->pos, ctype, SYM_ARRAY);
token = abstract_array_declarator(token->next, array);
token = expect(token, ']', "in abstract_array_declarator");
continue;
@@ -383,7 +384,7 @@ static struct token *direct_declarator(struct token *token, struct symbol **tree
if (token->special == ':') {
struct symbol *bitfield;
struct expression *expr;
- bitfield = indirect(token, ctype, SYM_BITFIELD);
+ bitfield = indirect(token->pos, ctype, SYM_BITFIELD);
token = conditional_expression(token->next, &expr);
bitfield->fieldwidth = get_expression_value(expr);
continue;
@@ -391,7 +392,6 @@ static struct token *direct_declarator(struct token *token, struct symbol **tree
break;
}
if (p) {
- (*tree)->token = *p;
(*tree)->ident = *p;
}
return token;
@@ -406,7 +406,7 @@ static struct token *pointer(struct token *token, struct ctype *ctype)
base_type = ctype->base_type;
while (match_op(token,'*')) {
- struct symbol *ptr = alloc_symbol(NULL, SYM_PTR);
+ struct symbol *ptr = alloc_symbol(token->pos, SYM_PTR);
ptr->ctype.modifiers = modifiers & ~MOD_STORAGE;
ptr->ctype.base_type = base_type;
@@ -420,7 +420,7 @@ static struct token *pointer(struct token *token, struct ctype *ctype)
return token;
}
-static struct token *declarator(struct token *token, struct symbol **tree, struct token **p)
+static struct token *declarator(struct token *token, struct symbol **tree, struct ident **p)
{
token = pointer(token, &(*tree)->ctype);
return direct_declarator(token, tree, p);
@@ -433,8 +433,8 @@ static struct token *struct_declaration_list(struct token *token, struct symbol_
token = declaration_specifiers(token, &ctype);
for (;;) {
- struct token *ident = NULL;
- struct symbol *decl = alloc_symbol(token, SYM_NODE);
+ struct ident *ident = NULL;
+ struct symbol *decl = alloc_symbol(token->pos, SYM_NODE);
decl->ctype = ctype;
token = pointer(token, &decl->ctype);
token = direct_declarator(token, &decl, &ident);
@@ -456,12 +456,12 @@ static struct token *struct_declaration_list(struct token *token, struct symbol_
static struct token *parameter_declaration(struct token *token, struct symbol **tree)
{
- struct token *ident = NULL;
+ struct ident *ident = NULL;
struct symbol *sym;
struct ctype ctype = { 0, };
token = declaration_specifiers(token, &ctype);
- sym = alloc_symbol(token, SYM_NODE);
+ sym = alloc_symbol(token->pos, SYM_NODE);
sym->ctype = ctype;
*tree = sym;
token = pointer(token, &sym->ctype);
@@ -471,7 +471,7 @@ static struct token *parameter_declaration(struct token *token, struct symbol **
struct token *typename(struct token *token, struct symbol **p)
{
- struct symbol *sym = alloc_symbol(token, SYM_NODE);
+ struct symbol *sym = alloc_symbol(token->pos, SYM_NODE);
*p = sym;
token = declaration_specifiers(token, &sym->ctype);
return declarator(token, &sym, NULL);
@@ -524,12 +524,12 @@ struct token *statement(struct token *token, struct statement **tree)
struct statement *stmt = alloc_statement(token, STMT_NONE);
*tree = stmt;
- if (token->type == TOKEN_IDENT) {
+ if (token_type(token) == TOKEN_IDENT) {
if (token->ident == &if_ident) {
stmt->type = STMT_IF;
token = parens_expression(token->next, &stmt->if_conditional, "after if");
token = statement(token, &stmt->if_true);
- if (token->type != TOKEN_IDENT)
+ if (token_type(token) != TOKEN_IDENT)
return token;
if (token->ident != &else_ident)
return token;
@@ -606,10 +606,10 @@ default_statement:
struct statement *iterator;
token = statement(token->next, &iterator);
- if (token->type == TOKEN_IDENT && token->ident == &while_ident)
+ if (token_type(token) == TOKEN_IDENT && token->ident == &while_ident)
token = token->next;
else
- warn(token, "expected 'while' after 'do'");
+ warn(token->pos, "expected 'while' after 'do'");
token = parens_expression(token, &expr, "after 'do-while'");
stmt->type = STMT_ITERATOR;
@@ -621,18 +621,18 @@ default_statement:
if (token->ident == &goto_ident) {
stmt->type = STMT_GOTO;
token = token->next;
- if (token->type == TOKEN_IDENT) {
+ if (token_type(token) == TOKEN_IDENT) {
stmt->goto_label = token;
token = token->next;
} else
- warn(token, "invalid label");
+ warn(token->pos, "invalid label");
return expect(token, ';', "at end of statement");
}
if (token->ident == &asm_ident || token->ident == &__asm___ident || token->ident == &__asm_ident) {
struct expression *expr;
stmt->type = STMT_ASM;
token = token->next;
- if (token->type == TOKEN_IDENT) {
+ if (token_type(token) == TOKEN_IDENT) {
if (token->ident == &__volatile___ident || token->ident == &volatile_ident)
token = token->next;
}
@@ -685,7 +685,7 @@ static struct token *parameter_type_list(struct token *token, struct symbol *fn)
{
struct symbol_list **list = &fn->arguments;
for (;;) {
- struct symbol *sym = alloc_symbol(token, SYM_NODE);
+ struct symbol *sym = alloc_symbol(token->pos, SYM_NODE);
if (match_op(token, SPECIAL_ELLIPSIS)) {
fn->variadic = 1;
@@ -695,7 +695,7 @@ static struct token *parameter_type_list(struct token *token, struct symbol *fn)
token = parameter_declaration(token, &sym);
/* Special case: (void) */
- if (!*list && !sym->token && sym->ctype.base_type == &void_ctype)
+ if (!*list && !sym->ident && sym->ctype.base_type == &void_ctype)
break;
add_symbol(list, sym);
if (!match_op(token, ','))
@@ -743,10 +743,10 @@ struct token *initializer(struct token *token, struct ctype *type)
struct token *next, *name = NULL;
next = token->next;
- if (match_op(token, '.') && (next->type == TOKEN_IDENT) && match_op(next->next, '=')) {
+ if (match_op(token, '.') && (token_type(next) == TOKEN_IDENT) && match_op(next->next, '=')) {
name = next;
token = next->next->next;
- } else if ((token->type == TOKEN_IDENT) && match_op(next, ':')) {
+ } else if ((token_type(token) == TOKEN_IDENT) && match_op(next, ':')) {
name = token;
token = next->next;
}
@@ -763,22 +763,22 @@ static void declare_argument(struct symbol *sym, void *data, int flags)
struct symbol *decl = data;
if (!sym->ident) {
- warn(decl->token, "no identifier for function argument");
+ warn(decl->pos, "no identifier for function argument");
return;
}
- bind_symbol(sym, sym->ident->ident, NS_SYMBOL);
+ bind_symbol(sym, sym->ident, NS_SYMBOL);
}
static struct token *external_declaration(struct token *token, struct symbol_list **list)
{
- struct token *ident = NULL;
+ struct ident *ident = NULL;
struct symbol *decl;
struct ctype ctype = { 0, };
struct symbol *base_type;
/* Parse declaration-specifiers, if any */
token = declaration_specifiers(token, &ctype);
- decl = alloc_symbol(token, SYM_NODE);
+ decl = alloc_symbol(token->pos, SYM_NODE);
decl->ctype = ctype;
token = pointer(token, &decl->ctype);
token = declarator(token, &decl, &ident);
@@ -791,10 +791,10 @@ static struct token *external_declaration(struct token *token, struct symbol_lis
/* type define declaration? */
if (ctype.modifiers & MOD_TYPEDEF) {
- bind_symbol(decl, ident->ident, NS_TYPEDEF);
+ bind_symbol(decl, ident, NS_TYPEDEF);
} else {
add_symbol(list, decl);
- bind_symbol(decl, ident->ident, NS_SYMBOL);
+ bind_symbol(decl, ident, NS_SYMBOL);
}
base_type = decl->ctype.base_type;
@@ -814,20 +814,20 @@ static struct token *external_declaration(struct token *token, struct symbol_lis
break;
ident = NULL;
- decl = alloc_symbol(token, SYM_NODE);
+ decl = alloc_symbol(token->pos, SYM_NODE);
decl->ctype = ctype;
token = pointer(token, &decl->ctype);
token = declarator(token->next, &decl, &ident);
if (!ident) {
- warn(token, "expected identifier name in type definition");
+ warn(token->pos, "expected identifier name in type definition");
return token;
}
if (ctype.modifiers & MOD_TYPEDEF) {
- bind_symbol(decl, ident->ident, NS_TYPEDEF);
+ bind_symbol(decl, ident, NS_TYPEDEF);
} else {
add_symbol(list, decl);
- bind_symbol(decl, ident->ident, NS_SYMBOL);
+ bind_symbol(decl, ident, NS_SYMBOL);
}
}
return expect(token, ';', "at end of declaration");
diff --git a/pre-process.c b/pre-process.c
index 5e3f82a..748dabf 100644
--- a/pre-process.c
+++ b/pre-process.c
@@ -49,14 +49,14 @@ static int match_string_ident(struct ident *ident, const char *str)
return !str[ident->len] && !memcmp(str, ident->name, ident->len);
}
-static struct token *alloc_token(struct token *dup)
+static struct token *alloc_token(struct position *pos)
{
struct token *token = __alloc_token(0);
- token->stream = dup->stream;
- token->line = dup->line;
- token->pos = dup->pos;
- token->whitespace = 1;
+ token->pos.stream = pos->stream;
+ token->pos.line = pos->line;
+ token->pos.pos = pos->pos;
+ token->pos.whitespace = 1;
return token;
}
@@ -72,7 +72,7 @@ static struct token *for_each_ident(struct token *head, struct token *(*action)(
if (eof_token(next))
break;
- if (next->type == TOKEN_IDENT)
+ if (token_type(next) == TOKEN_IDENT)
next = action(head, next);
head = next;
@@ -84,9 +84,9 @@ static struct token *is_defined(struct token *head, struct token *token, struct
{
char *string[] = { "0", "1" };
char *defined = string[lookup_symbol(token->ident, NS_PREPROCESSOR) != NULL];
- struct token *newtoken = alloc_token(token);
+ struct token *newtoken = alloc_token(&token->pos);
- newtoken->type = TOKEN_INTEGER;
+ token_type(newtoken) = TOKEN_INTEGER;
newtoken->integer = defined;
newtoken->next = next;
head->next = newtoken;
@@ -107,7 +107,7 @@ struct token *defined_one_symbol(struct token *head, struct token *next)
return next;
past = past->next;
}
- if (token->type == TOKEN_IDENT)
+ if (token_type(token) == TOKEN_IDENT)
return is_defined(head, token, past);
}
return next;
@@ -128,7 +128,7 @@ static void replace_with_string(struct token *token, const char *str)
s->length = size;
memcpy(s->data, str, size);
- token->type = TOKEN_STRING;
+ token_type(token) = TOKEN_STRING;
token->string = s;
}
@@ -136,7 +136,7 @@ static void replace_with_integer(struct token *token, unsigned int val)
{
char *buf = __alloc_bytes(10);
sprintf(buf, "%d", val);
- token->type = TOKEN_INTEGER;
+ token_type(token) = TOKEN_INTEGER;
token->integer = buf;
}
@@ -149,9 +149,9 @@ struct token *expand_one_symbol(struct token *head, struct token *token)
return expand(head, sym);
}
if (!memcmp(token->ident->name, "__LINE__", 9)) {
- replace_with_integer(token, token->line);
+ replace_with_integer(token, token->pos.line);
} else if (!memcmp(token->ident->name, "__FILE__", 9)) {
- replace_with_string(token, (input_streams + token->stream)->name);
+ replace_with_string(token, (input_streams + token->pos.stream)->name);
}
return token;
}
@@ -181,12 +181,12 @@ static struct token *find_argument_end(struct token *start)
return start;
}
-static struct token *dup_token(struct token *token, struct token *pos, int newline)
+static struct token *dup_token(struct token *token, struct position *pos, int newline)
{
struct token *alloc = alloc_token(pos);
- alloc->type = token->type;
- alloc->line = pos->line;
- alloc->newline = newline;
+ token_type(alloc) = token_type(token);
+ alloc->pos.line = pos->line;
+ alloc->pos.newline = newline;
alloc->integer = token->integer;
return alloc;
}
@@ -199,11 +199,11 @@ static void insert(struct token *token, struct token *prev)
static struct token * replace(struct token *token, struct token *prev, struct token *list)
{
- int newline = token->newline;
+ int newline = token->pos.newline;
prev->next = token->next;
while (!eof_token(list) && !match_op(list, SPECIAL_ARG_SEPARATOR)) {
- struct token *newtok = dup_token(list, token, newline);
+ struct token *newtok = dup_token(list, &token->pos, newline);
newline = 0;
insert(newtok, prev);
prev = newtok;
@@ -230,13 +230,13 @@ static struct token *stringify(struct token *token, struct token *arg)
{
const char *s = show_token_sequence(arg);
int size = strlen(s)+1;
- struct token *newtoken = alloc_token(token);
+ struct token *newtoken = alloc_token(&token->pos);
struct string *string = __alloc_string(size);
- newtoken->newline = token->newline;
+ newtoken->pos.newline = token->pos.newline;
memcpy(string->data, s, size);
string->length = size;
- newtoken->type = TOKEN_STRING;
+ token_type(newtoken) = TOKEN_STRING;
newtoken->string = string;
newtoken->next = &eof_token_entry;
return newtoken;
@@ -255,7 +255,7 @@ static int arg_number(struct token *arglist, struct ident *ident)
return -1;
}
-static struct token empty_arg_token = { .type = TOKEN_EOF };
+static struct token empty_arg_token = { .pos = { .type = TOKEN_EOF } };
static struct token *expand_one_arg(struct token *head, struct token *token,
struct token *arglist, struct token *arguments)
@@ -299,15 +299,15 @@ static void expand_arguments(struct token *token, struct token *head,
if (match_op(next, '#')) {
struct token *nextnext = next->next;
int nr = arg_number(arglist, nextnext->ident);
- if (nextnext != head && nr >= 0 && nextnext->type == TOKEN_IDENT) {
+ if (nextnext != head && nr >= 0 && token_type(nextnext) == TOKEN_IDENT) {
struct token *newtoken = stringify(nextnext, get_argument(nr, arguments));
replace(nextnext, head, newtoken);
continue;
}
- warn(next, "'#' operation is not followed by argument name");
+ warn(next->pos, "'#' operation is not followed by argument name");
}
- if (next->type == TOKEN_IDENT)
+ if (token_type(next) == TOKEN_IDENT)
next = expand_one_arg(head, next, arglist, arguments);
head = next;
@@ -339,13 +339,13 @@ static struct token *hashhash(struct token *head, struct token *first, struct to
*
* See expand_one_arg.
*/
- if (second->type == TOKEN_EOF) {
+ if (token_type(second) == TOKEN_EOF) {
head->next = second->next;
return head;
}
p = buffer;
- switch (first->type) {
+ switch (token_type(first)) {
case TOKEN_INTEGER:
len = strlen(first->integer);
src = first->integer;
@@ -360,7 +360,7 @@ static struct token *hashhash(struct token *head, struct token *first, struct to
memcpy(p, src, len);
p += len;
- switch (second->type) {
+ switch (token_type(second)) {
case TOKEN_INTEGER:
len = strlen(second->integer);
src = second->integer;
@@ -376,10 +376,10 @@ static struct token *hashhash(struct token *head, struct token *first, struct to
p += len;
*p++ = 0;
- newtoken = alloc_token(first);
+ newtoken = alloc_token(&first->pos);
head->next = newtoken;
- newtoken->type = first->type;
- switch (newtoken->type) {
+ token_type(newtoken) = token_type(first);
+ switch (token_type(newtoken)) {
case TOKEN_IDENT:
newtoken->ident = built_in_ident(buffer);
break;
@@ -468,7 +468,7 @@ static const char *token_name_sequence(struct token *token, int endop, struct to
while (!eof_token(token) && !match_op(token, endop)) {
int len;
const char *val = token->string->data;
- if (token->type != TOKEN_STRING)
+ if (token_type(token) != TOKEN_STRING)
val = show_token(token);
len = strlen(val);
memcpy(ptr, val, len);
@@ -477,7 +477,7 @@ static const char *token_name_sequence(struct token *token, int endop, struct to
}
*ptr = 0;
if (endop && !match_op(token, endop))
- warn(start, "expected '>' at end of filename");
+ warn(start->pos, "expected '>' at end of filename");
return buffer;
}
@@ -501,7 +501,7 @@ static void do_include(struct token *head, struct token *token, const char *file
return;
}
}
- warn(token, "unable to open '%s'", filename);
+ warn(token->pos, "unable to open '%s'", filename);
}
static int handle_include(struct stream *stream, struct token *head, struct token *token)
@@ -534,7 +534,7 @@ static int token_list_different(struct token *list1, struct token *list2)
return 0;
if (!list1 || !list2)
return 1;
- if (list1->type != list2->type)
+ if (token_type(list1) != token_type(list2))
return 1;
list1 = list1->next;
list2 = list2->next;
@@ -549,8 +549,8 @@ static int handle_define(struct stream *stream, struct token *head, struct token
struct symbol *sym;
struct ident *name;
- if (left->type != TOKEN_IDENT) {
- warn(head, "expected identifier to 'define'");
+ if (token_type(left) != TOKEN_IDENT) {
+ warn(head->pos, "expected identifier to 'define'");
return 0;
}
if (false_nesting)
@@ -559,7 +559,7 @@ static int handle_define(struct stream *stream, struct token *head, struct token
arglist = NULL;
expansion = left->next;
- if (!expansion->whitespace && match_op(expansion, '(')) {
+ if (!expansion->pos.whitespace && match_op(expansion, '(')) {
arglist = expansion;
while (!eof_token(expansion)) {
struct token *next = expansion->next;
@@ -580,12 +580,12 @@ static int handle_define(struct stream *stream, struct token *head, struct token
if (sym) {
if (token_list_different(sym->expansion, expansion) ||
token_list_different(sym->arglist, arglist)) {
- warn(left, "preprocessor token redefined");
- warn(sym->token, "this was the original definition");
+ warn(left->pos, "preprocessor token redefined");
+ warn(sym->pos, "this was the original definition");
}
return 1;
}
- sym = alloc_symbol(left, SYM_NODE);
+ sym = alloc_symbol(left->pos, SYM_NODE);
bind_symbol(sym, name, NS_PREPROCESSOR);
sym->expansion = expansion;
@@ -598,8 +598,8 @@ static int handle_undef(struct stream *stream, struct token *head, struct token
struct token *left = token->next;
struct symbol **sym;
- if (left->type != TOKEN_IDENT) {
- warn(head, "expected identifier to 'undef'");
+ if (token_type(left) != TOKEN_IDENT) {
+ warn(head->pos, "expected identifier to 'undef'");
return 0;
}
if (false_nesting)
@@ -631,10 +631,10 @@ static int preprocessor_if(struct token *token, int true)
static int token_defined(struct token *token)
{
- if (token->type == TOKEN_IDENT)
+ if (token_type(token) == TOKEN_IDENT)
return lookup_symbol(token->ident, NS_PREPROCESSOR) != NULL;
- warn(token, "expected identifier for #if[n]def");
+ warn(token->pos, "expected identifier for #if[n]def");
return 0;
}
@@ -648,7 +648,7 @@ static int handle_ifndef(struct stream *stream, struct token *head, struct token
struct token *next = token->next;
if (stream->constant == -1) {
int newconstant = 0;
- if (next->type == TOKEN_IDENT) {
+ if (token_type(next) == TOKEN_IDENT) {
if (!stream->protect || stream->protect == next->ident) {
newconstant = -2;
stream->protect = next->ident;
@@ -670,7 +670,7 @@ static int expression_value(struct token *head)
expand_list(head);
token = constant_expression(head->next, &expr);
if (!eof_token(token))
- warn(token, "garbage at end: %s", show_token_sequence(token));
+ warn(token->pos, "garbage at end: %s", show_token_sequence(token));
value = get_expression_value(expr);
return value != 0;
}
@@ -703,7 +703,7 @@ static int handle_elif(struct stream * stream, struct token *head, struct token
true_nesting--;
return 1;
}
- warn(token, "unmatched '#elif'");
+ warn(token->pos, "unmatched '#elif'");
return 1;
}
@@ -725,7 +725,7 @@ static int handle_else(struct stream *stream, struct token *head, struct token *
false_nesting = 1;
return 1;
}
- warn(token, "unmatched #else");
+ warn(token->pos, "unmatched #else");
return 1;
}
@@ -742,7 +742,7 @@ static int handle_endif(struct stream *stream, struct token *head, struct token
true_nesting--;
return 1;
}
- warn(token, "unmatched #endif");
+ warn(token->pos, "unmatched #endif");
return 1;
}
@@ -762,7 +762,7 @@ static const char *show_token_sequence(struct token *token)
memcpy(ptr, val, len);
ptr += len;
token = token->next;
- whitespace = token->whitespace;
+ whitespace = token->pos.whitespace;
}
*ptr = 0;
return buffer;
@@ -772,7 +772,7 @@ static int handle_warning(struct stream *stream, struct token *head, struct toke
{
if (false_nesting)
return 1;
- warn(token, "%s", show_token_sequence(token->next));
+ warn(token->pos, "%s", show_token_sequence(token->next));
return 1;
}
@@ -780,7 +780,7 @@ static int handle_error(struct stream *stream, struct token *head, struct token
{
if (false_nesting)
return 1;
- error(token, "%s", show_token_sequence(token->next));
+ error(token->pos, "%s", show_token_sequence(token->next));
return 1;
}
@@ -803,7 +803,7 @@ static void add_path_entry(struct token *token, const char *path)
return;
}
}
- warn(token, "too many include path entries");
+ warn(token->pos, "too many include path entries");
}
static int handle_add_include(struct stream *stream, struct token *head, struct token *token)
@@ -812,8 +812,8 @@ static int handle_add_include(struct stream *stream, struct token *head, struct
token = token->next;
if (eof_token(token))
return 1;
- if (token->type != TOKEN_STRING) {
- warn(token, "expected path string");
+ if (token_type(token) != TOKEN_STRING) {
+ warn(token->pos, "expected path string");
return 1;
}
add_path_entry(token, token->string->data);
@@ -856,10 +856,10 @@ static void handle_preprocessor_line(struct stream *stream, struct token * head,
if (!token)
return;
- if (token->type == TOKEN_IDENT)
+ if (token_type(token) == TOKEN_IDENT)
if (handle_preprocessor_command(stream, head, token->ident, token))
return;
- warn(token, "unrecognized preprocessor line '%s'", show_token_sequence(token));
+ warn(token->pos, "unrecognized preprocessor line '%s'", show_token_sequence(token));
}
static void preprocessor_line(struct stream *stream, struct token * head)
@@ -869,7 +869,7 @@ static void preprocessor_line(struct stream *stream, struct token * head)
for (;;) {
next = *tp;
- if (next->newline)
+ if (next->pos.newline)
break;
tp = &next->next;
}
@@ -882,9 +882,9 @@ static void do_preprocess(struct token *head)
{
do {
struct token *next = head->next;
- struct stream *stream = input_streams + next->stream;
+ struct stream *stream = input_streams + next->pos.stream;
- if (next->newline && match_op(next, '#')) {
+ if (next->pos.newline && match_op(next, '#')) {
preprocessor_line(stream, head);
continue;
}
@@ -894,7 +894,7 @@ static void do_preprocess(struct token *head)
continue;
}
- switch (next->type) {
+ switch (token_type(next)) {
case TOKEN_STREAMEND:
if (stream->constant == -1 && stream->protect) {
stream->constant = 1;
@@ -922,12 +922,12 @@ static void do_preprocess(struct token *head)
struct token * preprocess(struct token *token)
{
- struct token header = { 0, };
+ struct token header = { };
header.next = token;
do_preprocess(&header);
if (if_nesting)
- warn(unmatched_if, "unmatched preprocessor conditional");
+ warn(unmatched_if->pos, "unmatched preprocessor conditional");
// Drop all expressions from pre-processing, they're not used any more.
clear_expression_alloc();
diff --git a/show-parse.c b/show-parse.c
index 9c84325..82e6bd0 100644
--- a/show-parse.c
+++ b/show-parse.c
@@ -54,7 +54,7 @@ void show_struct_member(struct symbol *sym, void *data, int flags)
{
if (flags & ITERATE_FIRST)
printf(" { ");
- printf("%s:%d:%d at offset %ld", show_token(sym->token), sym->bit_size, sym->alignment, sym->offset);
+ printf("%s:%d:%d at offset %ld", show_ident(sym->ident), sym->bit_size, sym->alignment, sym->offset);
if (sym->fieldwidth)
printf("[%d..%d]", sym->bit_offset, sym->bit_offset+sym->fieldwidth-1);
if (flags & ITERATE_LAST)
@@ -145,15 +145,15 @@ void show_type(struct symbol *sym)
return;
case SYM_STRUCT:
- printf("struct %s", show_token(sym->token));
+ printf("struct %s", show_ident(sym->ident));
return;
case SYM_UNION:
- printf("union %s", show_token(sym->token));
+ printf("union %s", show_ident(sym->ident));
return;
case SYM_ENUM:
- printf("enum %s", show_token(sym->token));
+ printf("enum %s", show_ident(sym->ident));
return;
case SYM_NODE: {
@@ -162,7 +162,7 @@ void show_type(struct symbol *sym)
printf("notype");
else
show_type(type);
- printf(": %s", show_token(sym->token));
+ printf(": %s", show_ident(sym->ident));
return;
}
@@ -172,7 +172,7 @@ void show_type(struct symbol *sym)
return;
default:
- printf("strange type %d '%s' of type ", sym->type, show_token(sym->token));
+ printf("strange type %d '%s' of type ", sym->type, show_ident(sym->ident));
show_type(sym->ctype.base_type);
return;
}
@@ -423,11 +423,11 @@ void show_expression(struct expression *expr)
break;
case EXPR_SYMBOL:
if (!expr->symbol) {
- warn(expr->token, "undefined symbol '%s'", show_token(expr->token));
+ warn(expr->pos, "undefined symbol '%s'", show_ident(expr->symbol_name));
printf("<nosymbol>");
break;
}
- printf("<%s:", show_token(expr->symbol->token));
+ printf("<%s:", show_ident(expr->symbol_name));
show_type(expr->symbol->ctype.base_type);
printf(">");
break;
diff --git a/symbol.c b/symbol.c
index fba6ba6..159fb4f 100644
--- a/symbol.c
+++ b/symbol.c
@@ -27,11 +27,11 @@ struct symbol *lookup_symbol(struct ident *ident, enum namespace ns)
return sym;
}
-struct symbol *alloc_symbol(struct token *token, int type)
+struct symbol *alloc_symbol(struct position pos, int type)
{
struct symbol *sym = __alloc_symbol(0);
sym->type = type;
- sym->token = token;
+ sym->pos = pos;
return sym;
}
@@ -141,7 +141,7 @@ static void examine_bitfield_type(struct symbol *sym)
examine_symbol_type(base_type);
bit_size = base_type->bit_size;
if (sym->fieldwidth > bit_size) {
- warn(sym->token, "impossible field-width for this type");
+ warn(sym->pos, "impossible field-width for this type");
sym->fieldwidth = bit_size;
}
alignment = base_type->alignment;
@@ -223,7 +223,7 @@ void examine_symbol_type(struct symbol * sym)
void bind_symbol(struct symbol *sym, struct ident *ident, enum namespace ns)
{
if (sym->id_list) {
- warn(sym->token, "internal error: symbol type already bound");
+ warn(sym->pos, "internal error: symbol type already bound");
return;
}
sym->namespace = ns;
@@ -236,7 +236,7 @@ void bind_symbol(struct symbol *sym, struct ident *ident, enum namespace ns)
struct symbol *create_symbol(int stream, const char *name, int type)
{
struct token *token = built_in_token(stream, name);
- struct symbol *sym = alloc_symbol(token, type);
+ struct symbol *sym = alloc_symbol(token->pos, type);
bind_symbol(sym, token->ident, NS_TYPEDEF);
return sym;
}
diff --git a/symbol.h b/symbol.h
index 9d5432e..46760ec 100644
--- a/symbol.h
+++ b/symbol.h
@@ -52,8 +52,8 @@ struct ctype {
struct symbol {
enum namespace namespace:8;
enum type type:8;
- struct token *token; /* Where this symbol was declared */
- struct token *ident; /* What identifier this symbol is associated with */
+ struct position pos; /* Where this symbol was declared */
+ struct ident *ident; /* What identifier this symbol is associated with */
struct symbol *next_id; /* Next semantic symbol that shares this identifier */
struct symbol **id_list; /* Backpointer to symbol list head */
union {
@@ -155,7 +155,7 @@ extern struct ident __asm___ident,
extern struct symbol *lookup_symbol(struct ident *, enum namespace);
extern void init_symbols(void);
-extern struct symbol *alloc_symbol(struct token *, int type);
+extern struct symbol *alloc_symbol(struct position, int type);
extern void show_type(struct symbol *);
extern const char *modifier_string(unsigned long mod);
extern void show_symbol(struct symbol *);
diff --git a/test-lexing.c b/test-lexing.c
index 1cd82c9..9af877e 100644
--- a/test-lexing.c
+++ b/test-lexing.c
@@ -31,9 +31,9 @@ int main(int argc, char **argv)
while (!eof_token(token)) {
struct token *next = token->next;
char * separator = "";
- if (next->whitespace)
+ if (next->pos.whitespace)
separator = " ";
- if (next->newline)
+ if (next->pos.newline)
separator = "\n";
printf("%s%s", show_token(token), separator);
token = next;
diff --git a/token.h b/token.h
index 7c8ec69..8930575 100644
--- a/token.h
+++ b/token.h
@@ -9,6 +9,7 @@
*/
#include <sys/types.h>
+#include "lib.h"
/*
* This describes the pure lexical elements (tokens), with
@@ -114,12 +115,7 @@ struct string {
* pointers.
*/
struct token {
- unsigned int type:8,
- stream:8,
- pos:14,
- newline:1,
- whitespace:1;
- unsigned int line;
+ struct position pos;
struct token *next;
union {
char *integer;
@@ -131,6 +127,8 @@ struct token {
};
};
+#define token_type(x) ((x)->pos.type)
+
/*
* Last token in the stream - points to itself.
* This allows us to not test for NULL pointers
@@ -155,12 +153,12 @@ extern struct token *preprocess(struct token *);
static inline int match_op(struct token *token, int op)
{
- return token->type == TOKEN_SPECIAL && token->special == op;
+ return token->pos.type == TOKEN_SPECIAL && token->special == op;
}
static inline int match_ident(struct token *token, struct ident *id)
{
- return token->type == TOKEN_IDENT && token->ident == id;
+ return token->pos.type == TOKEN_IDENT && token->ident == id;
}
#endif
diff --git a/tokenize.c b/tokenize.c
index 9fa96d6..8169ec6 100644
--- a/tokenize.c
+++ b/tokenize.c
@@ -26,8 +26,8 @@ static int input_streams_allocated;
#define BUFSIZE (8192)
typedef struct {
- int fd, stream, line, pos, offset, size;
- unsigned int newline:1, whitespace:1;
+ int fd, offset, size;
+ struct position pos;
struct token **tokenlist;
struct token *token;
unsigned char *buffer;
@@ -49,6 +49,8 @@ const char *show_special(int val)
const char *show_ident(const struct ident *ident)
{
static char buffer[256];
+ if (!ident)
+ return "<noident>";
sprintf(buffer, "%.*s", ident->len, ident->name);
return buffer;
}
@@ -82,7 +84,7 @@ const char *show_token(const struct token *token)
if (!token)
return "<no token>";
- switch (token->type) {
+ switch (token_type(token)) {
case TOKEN_ERROR:
return "syntax error";
@@ -138,11 +140,11 @@ const char *show_token(const struct token *token)
}
case TOKEN_STREAMBEGIN:
- sprintf(buffer, "<beginning of '%s'>", (input_streams + token->stream)->name);
+ sprintf(buffer, "<beginning of '%s'>", (input_streams + token->pos.stream)->name);
return buffer;
case TOKEN_STREAMEND:
- sprintf(buffer, "<end of '%s'>", (input_streams + token->stream)->name);
+ sprintf(buffer, "<end of '%s'>", (input_streams + token->pos.stream)->name);
return buffer;
default:
@@ -189,11 +191,7 @@ int init_stream(const char *name, int fd)
static struct token * alloc_token(stream_t *stream)
{
struct token *token = __alloc_token(0);
- token->line = stream->line;
token->pos = stream->pos;
- token->stream = stream->stream;
- token->newline = stream->newline;
- token->whitespace = stream->whitespace;
return token;
}
@@ -213,11 +211,11 @@ static int nextchar(stream_t *stream)
}
c = stream->buffer[offset];
stream->offset = offset + 1;
- stream->pos++;
+ stream->pos.pos++;
if (c == '\n') {
- stream->line++;
- stream->newline = 1;
- stream->pos = 0;
+ stream->pos.line++;
+ stream->pos.newline = 1;
+ stream->pos.pos = 0;
}
return c;
}
@@ -229,11 +227,11 @@ static void mark_eof(stream_t *stream, struct token *end_token)
struct token *end;
end = alloc_token(stream);
- end->type = TOKEN_STREAMEND;
- end->newline = 1;
+ token_type(end) = TOKEN_STREAMEND;
+ end->pos.newline = 1;
eof_token_entry.next = &eof_token_entry;
- eof_token_entry.newline = 1;
+ eof_token_entry.pos.newline = 1;
if (!end_token)
end_token = &eof_token_entry;
@@ -254,8 +252,8 @@ static void add_token(stream_t *stream)
static void drop_token(stream_t *stream)
{
- stream->newline |= stream->token->newline;
- stream->whitespace |= stream->token->whitespace;
+ stream->pos.newline |= stream->token->pos.newline;
+ stream->pos.whitespace |= stream->token->pos.whitespace;
stream->token = NULL;
}
@@ -288,7 +286,7 @@ static int do_integer(char *buffer, int len, int next, stream_t *stream)
buffer[len++] = '\0';
buf = __alloc_bytes(len);
memcpy(buf, buffer, len);
- token->type = TOKEN_INTEGER;
+ token_type(token) = TOKEN_INTEGER;
token->integer = buf;
add_token(stream);
return next;
@@ -329,7 +327,7 @@ static int escapechar(int first, int type, stream_t *stream, int *valp)
value = first;
if (first == '\n')
- warn(stream->token, "Newline in string or character constant");
+ warn(stream->pos, "Newline in string or character constant");
if (first == '\\' && next != EOF) {
value = next;
@@ -375,7 +373,7 @@ static int escapechar(int first, int type, stream_t *stream, int *valp)
}
/* Fallthrough */
default:
- warn(stream->token, "Unknown escape '%c'", value);
+ warn(stream->pos, "Unknown escape '%c'", value);
}
}
/* Mark it as escaped */
@@ -392,13 +390,13 @@ static int get_char_token(int next, stream_t *stream)
next = escapechar(next, '\'', stream, &value);
if (value == '\'' || next != '\'') {
- warn(stream->token, "Bad character constant");
+ warn(stream->pos, "Bad character constant");
drop_token(stream);
return next;
}
token = stream->token;
- token->type = TOKEN_CHAR;
+ token_type(token) = TOKEN_CHAR;
token->character = value & 0xff;
add_token(stream);
@@ -418,7 +416,7 @@ static int get_string_token(int next, stream_t *stream)
if (val == '"')
break;
if (next == EOF) {
- warn(stream->token, "Enf of file in middle of string");
+ warn(stream->pos, "Enf of file in middle of string");
return next;
}
if (len < sizeof(buffer)) {
@@ -429,7 +427,7 @@ static int get_string_token(int next, stream_t *stream)
}
if (len > 256)
- warn(stream->token, "String too long");
+ warn(stream->pos, "String too long");
string = __alloc_string(len+1);
memcpy(string->data, buffer, len);
@@ -438,7 +436,7 @@ static int get_string_token(int next, stream_t *stream)
/* Pass it on.. */
token = stream->token;
- token->type = TOKEN_STRING;
+ token_type(token) = TOKEN_STRING;
token->string = string;
add_token(stream);
@@ -466,7 +464,7 @@ static int drop_stream_comment(stream_t *stream)
for (;;) {
int curr = next;
if (curr == EOF) {
- warn(stream->token, "End of file in the middle of a comment");
+ warn(stream->pos, "End of file in the middle of a comment");
return curr;
}
next = nextchar(stream);
@@ -523,7 +521,7 @@ static int get_one_special(int c, stream_t *stream)
/* Pass it on.. */
token = stream->token;
- token->type = TOKEN_SPECIAL;
+ token_type(token) = TOKEN_SPECIAL;
token->special = value;
add_token(stream);
return next;
@@ -632,8 +630,8 @@ struct token *built_in_token(int stream, const char *name)
struct token *token;
token = __alloc_token(0);
- token->stream = stream;
- token->type = TOKEN_IDENT;
+ token->pos.stream = stream;
+ token_type(token) = TOKEN_IDENT;
token->ident = built_in_ident(name);
return token;
}
@@ -671,7 +669,7 @@ static int get_one_identifier(int c, stream_t *stream)
/* Pass it on.. */
token = stream->token;
- token->type = TOKEN_IDENT;
+ token_type(token) = TOKEN_IDENT;
token->ident = ident;
add_token(stream);
return next;
@@ -696,19 +694,20 @@ static struct token *setup_stream(stream_t *stream, int idx, int fd,
{
struct token *begin;
- stream->stream = idx;
+ stream->pos.stream = idx;
+ stream->pos.line = 1;
+ stream->pos.newline = 1;
+ stream->pos.whitespace = 0;
+ stream->pos.pos = 0;
+
stream->token = NULL;
- stream->line = 1;
- stream->newline = 1;
- stream->whitespace = 0;
- stream->pos = 0;
stream->fd = fd;
stream->offset = 0;
stream->size = buf_size;
stream->buffer = buf;
begin = alloc_token(stream);
- begin->type = TOKEN_STREAMBEGIN;
+ token_type(begin) = TOKEN_STREAMBEGIN;
stream->tokenlist = &begin->next;
return begin;
}
@@ -719,21 +718,19 @@ static void tokenize_stream(stream_t *stream, struct token *endtoken)
while (c != EOF) {
if (c == '\\') {
c = nextchar(stream);
- stream->newline = 0;
- stream->whitespace = 1;
+ stream->pos.newline = 0;
+ stream->pos.whitespace = 1;
continue;
}
if (!isspace(c)) {
struct token *token = alloc_token(stream);
- token->newline = stream->newline;
- token->whitespace = stream->whitespace;
- stream->newline = 0;
- stream->whitespace = 0;
stream->token = token;
+ stream->pos.newline = 0;
+ stream->pos.whitespace = 0;
c = get_one_token(c, stream);
continue;
}
- stream->whitespace = 1;
+ stream->pos.whitespace = 1;
c = nextchar(stream);
}
mark_eof(stream, endtoken);