Changeset 0662451 in mainline for uspace/app/bdsh/tok.c

Timestamp:

2011-08-19T14:44:49Z (13 years ago)

Author:

Martin Sucha <sucha14@…>

Branches:

lfn, master, serial, ticket/834-toolchain-update, topic/msim-upgrade, topic/simplify-dev-export

Children:

5992e0e

Parents:

89660f2

Message:

Extend bdsh tokenizer to include more information

File:

• uspace/app/bdsh/tok.c (modified) (12 diffs)

uspace/app/bdsh/tok.c

@@ -42 +42 @@
 static bool tok_pending_chars(tokenizer_t *);
 static int tok_finish_string(tokenizer_t *);
+static void tok_start_token(tokenizer_t *, token_type_t);
 
 /** Initialize the token parser
@@ -61 +62 @@
         tok->outtok = out_tokens;
         tok->outtok_offset = 0;
-        /* Leave one slot for a null terminator */
-        assert(max_tokens > 0);
-        tok->outtok_size = max_tokens - 1;
+        tok->outtok_size = max_tokens;
         
         /* Prepare a buffer where all the token strings will be stored */
@@ -90 +89 @@
 
 /** Tokenize the input string into the tokens */
-int tok_tokenize(tokenizer_t *tok)
+int tok_tokenize(tokenizer_t *tok, size_t *tokens_length)
 {
         int rc;
@@ -98 +97 @@
         while ((cur_char = tok_get_char(tok)) != 0) {
                 if (cur_char == ' ') {
-                        /* Spaces delimit tokens, but are not processed in any way
-                         * Push the token if there is any.
+                        /* Push the token if there is any.
                          * There may not be any pending char for a token in case
                          * there are several spaces in the input.
@@ -109 +107 @@
                                 }
                         }
+                        tok_start_token(tok, TOKTYPE_SPACE);
+                        /* Eat all spaces */
+                        while (tok_look_char(tok) == ' ') {
+                                tok_push_char(tok, tok_get_char(tok));
+                        }
+                        tok_push_token(tok);
+                        
                 }
                 else if (cur_char == '|') {
@@ -121 +126 @@
                         }
                         
+                        tok_start_token(tok, TOKTYPE_PIPE);
+                        
                         rc = tok_push_char(tok, '|');
                         if (rc != EOK) {
@@ -135 +142 @@
                          * A literal quote is written as ''
                          */
+                        tok_start_token(tok, TOKTYPE_TEXT);
                         rc = tok_finish_string(tok);
                         if (rc != EOK) {
@@ -141 +149 @@
                 }
                 else {
+                        if (!tok_pending_chars(tok)) {
+                                tok_start_token(tok, TOKTYPE_TEXT);
+                        }
                         /* If we are handling any other character, just append it to
                          * the current token.
@@ -159 +170 @@
         }
         
-        /* We always have a space for the terminator, as we
-         * reserved it in tok_init */
-        tok->outtok[tok->outtok_offset] = 0;
+        *tokens_length = tok->outtok_offset;
         
         return EOK;
@@ -211 +220 @@
 wchar_t tok_look_char(tokenizer_t *tok)
 {
-        off_t old_offset = tok->in_offset;
-        off_t old_char_offset = tok->in_char_offset;
+        unsigned int old_offset = tok->in_offset;
+        unsigned int old_char_offset = tok->in_char_offset;
         wchar_t ret = tok_get_char(tok);
         tok->in_offset = old_offset;
@@ -225 +234 @@
 }
 
+void tok_start_token(tokenizer_t *tok, token_type_t type)
+{
+        tok->current_type = type;
+}
+
 /** Push the current token to the output array */
 int tok_push_token(tokenizer_t *tok)
@@ -237 +251 @@
         
         tok->outbuf[tok->outbuf_offset++] = 0;
-        token_t *tokinfo = &tok->outtok[tok->outtok_offset++]
-        tokinfo.text = tok->outbuf + tok->outbuf_last_start;
-        tokinfo.byte_start = tok->last_in_offset;
-        tokinfo.byte_length = tok->in_offset - tok->last_in_offset - 1;
-        tokinfo.char_start = tok->last_in_char_offset;
-        tokinfo.char_length = tok->in_char_offset - tok->last_in_char_offset
+        token_t *tokinfo = &tok->outtok[tok->outtok_offset++];
+        tokinfo->type = tok->current_type;
+        tokinfo->text = tok->outbuf + tok->outbuf_last_start;
+        tokinfo->byte_start = tok->last_in_offset;
+        tokinfo->byte_length = tok->in_offset - tok->last_in_offset - 1;
+        tokinfo->char_start = tok->last_in_char_offset;
+        tokinfo->char_length = tok->in_char_offset - tok->last_in_char_offset
             - 1;
-        tok->outtok[tok->outtok_offset]
         tok->outbuf_last_start = tok->outbuf_offset;