Changeset bd5f3b7 in mainline for uspace/app/bdsh

Timestamp:

2011-08-21T13:07:35Z (15 years ago)

Author:

Jakub Jermar <jakub@…>

Branches:

lfn, master, serial, ticket/834-toolchain-update, topic/fix-logger-deadlock, topic/msim-upgrade, topic/simplify-dev-export

Children:

00aece0, f1a9e87

Parents:

86a34d3e (diff), a6480d5 (diff)
Note: this is a merge changeset, the changes displayed below correspond to the merge itself.
Use the (diff) links above to see all the changes relative to each parent.

Message:

Merge mainline changes.

Location:

uspace/app/bdsh

Files:

• Makefile (modified) (2 diffs)
• cmds/modules/bdd/bdd.c (modified) (8 diffs)
• cmds/modules/cp/cp.c (modified) (2 diffs)
• cmds/modules/help/help.c (modified) (3 diffs)
• cmds/modules/ls/ls.c (modified) (1 diff)
• compl.c (added)
• compl.h (added)
• config.h (modified) (1 diff)
• exec.c (modified) (7 diffs)
• exec.h (modified) (1 diff)
• input.c (modified) (10 diffs)
• scli.h (modified) (1 diff)
• tok.c (modified) (14 diffs)
• tok.h (modified) (2 diffs)

uspace/app/bdsh/Makefile

@@ -29 +29 @@
 
 USPACE_PREFIX = ../..
-LIBS = $(LIBBLOCK_PREFIX)/libblock.a $(LIBCLUI_PREFIX)/libclui.a
-EXTRA_CFLAGS = -I$(LIBBLOCK_PREFIX) -I$(LIBCLUI_PREFIX) -I. -Icmds/ \
-        -Icmds/builtins -Icmds/modules
+LIBS = $(LIBBLOCK_PREFIX)/libblock.a $(LIBCLUI_PREFIX)/libclui.a \
+        $(LIBFMTUTIL_PREFIX)/libfmtutil.a
+EXTRA_CFLAGS = -I$(LIBBLOCK_PREFIX) -I$(LIBCLUI_PREFIX) -I$(LIBFMTUTIL_PREFIX)\
+        -I. -Icmds/ -Icmds/builtins -Icmds/modules
 BINARY = bdsh
 
@@ -54 +55 @@
         cmds/mod_cmds.c \
         cmds/builtin_cmds.c \
+        compl.c \
         errors.c \
         input.c \

uspace/app/bdsh/cmds/modules/bdd/bdd.c

@@ -39 +39 @@
 
 #include <libblock.h>
-#include <devmap.h>
+#include <loc.h>
 #include <errno.h>
 #include <assert.h>
@@ -69 +69 @@
         unsigned int argc;
         unsigned int i, j;
-        devmap_handle_t handle;
+        service_id_t service_id;
         aoff64_t offset;
         uint8_t *blk;
@@ -96 +96 @@
                 size = 256;
 
-        rc = devmap_device_get_handle(argv[1], &handle, 0);
+        rc = loc_service_get_id(argv[1], &service_id, 0);
         if (rc != EOK) {
                 printf("%s: Error resolving device `%s'.\n", cmdname, argv[1]);
@@ -102 +102 @@
         }
 
-        rc = block_init(EXCHANGE_SERIALIZE, handle, 2048);
+        rc = block_init(EXCHANGE_SERIALIZE, service_id, 2048);
         if (rc != EOK)  {
                 printf("%s: Error initializing libblock.\n", cmdname);
@@ -108 +108 @@
         }
 
-        rc = block_get_bsize(handle, &block_size);
+        rc = block_get_bsize(service_id, &block_size);
         if (rc != EOK) {
                 printf("%s: Error determining device block size.\n", cmdname);
@@ -117 +117 @@
         if (blk == NULL) {
                 printf("%s: Error allocating memory.\n", cmdname);
-                block_fini(handle);
+                block_fini(service_id);
                 return CMD_FAILURE;
         }
@@ -124 +124 @@
 
         while (size > 0) {
-                rc = block_read_direct(handle, ba, 1, blk);
+                rc = block_read_direct(service_id, ba, 1, blk);
                 if (rc != EOK) {
                         printf("%s: Error reading block %" PRIuOFF64 "\n", cmdname, ba);
                         free(blk);
-                        block_fini(handle);
+                        block_fini(service_id);
                         return CMD_FAILURE;
                 }
@@ -170 +170 @@
 
         free(blk);
-        block_fini(handle);
+        block_fini(service_id);
 
         return CMD_SUCCESS;

uspace/app/bdsh/cmds/modules/cp/cp.c

@@ -71 +71 @@
         size_t blen, int vb)
 {
-        int fd1, fd2, bytes = 0;
-        off64_t total = 0;
+        int fd1, fd2, bytes;
+        off64_t total;
         int64_t copied = 0;
         char *buff = NULL;
@@ -104 +104 @@
         }
 
-        for (;;) {
-                ssize_t res;
-                size_t written = 0;
-
-                bytes = read(fd1, buff, blen);
-                if (bytes <= 0)
+        while ((bytes = read_all(fd1, buff, blen)) > 0) {
+                if ((bytes = write_all(fd2, buff, bytes)) < 0)
                         break;
                 copied += bytes;
-                res = bytes;
-                do {
-                        /*
-                         * Theoretically, it may not be enough to call write()
-                         * only once. Also the previous read() may have
-                         * returned less data than requested.
-                         */
-                        bytes = write(fd2, buff + written, res);
-                        if (bytes < 0)
-                                goto err;
-                        written += bytes;
-                        res -= bytes;
-                } while (res > 0);
-
-                /* TODO: re-insert assert() once this is stand alone,
-                 * removed as abort() exits the entire shell
-                 */
-                if (res != 0) {
-                        printf("\n%zd more bytes than actually exist were copied\n", res);
-                        goto err;
-                }
         }
 
         if (bytes < 0) {
-err:
                 printf("\nError copying %s, (%d)\n", src, bytes);
                 copied = bytes;

uspace/app/bdsh/cmds/modules/help/help.c

@@ -1 +1 @@
 /*
  * Copyright (c) 2008 Tim Post
+ * Copyright (c) 2011 Martin Sucha
  * All rights reserved.
  *
@@ -30 +31 @@
 #include <stdlib.h>
 #include <str.h>
+#include <fmtutil.h>
 
 #include "config.h"
@@ -128 +130 @@
 static void help_survival(void)
 {
-        printf("Don't panic!\n\n");
-
-        printf("This is Bdsh, the Brain dead shell, currently "
+        print_wrapped_console(
+            "Don't panic!\n\n"
+
+            "This is Bdsh, the Brain dead shell, currently "
             "the primary user interface to HelenOS. Bdsh allows you to enter "
             "commands and supports history (Up, Down arrow keys), "
             "line editing (Left Arrow, Right Arrow, Home, End, Backspace), "
             "selection (Shift + movement keys), copy and paste (Ctrl-C, "
-            "Ctrl-V), similar to common desktop environments.\n\n");
-
-        printf("The most basic filesystem commands are Bdsh builtins. Type "
+            "Ctrl-V), similar to common desktop environments.\n\n"
+
+            "The most basic filesystem commands are Bdsh builtins. Type "
             "'help commands' [Enter] to see the list of Bdsh builtin commands. "
             "Other commands are external executables located in the /app and "
             "/srv directories. Type 'ls /app' [Enter] and 'ls /srv' [Enter] "
             "to see their list. You can execute an external command simply "
-            "by entering its name (e.g. type 'tetris' [Enter]).\n\n");
-
-        printf("HelenOS has virtual consoles (VCs). You can switch between "
-            "these using the F1-F11 keys.\n\n");
-
-        printf("This is but a small glimpse of what you can do with HelenOS. "
+            "by entering its name (e.g. type 'tetris' [Enter]).\n\n"
+
+            "HelenOS has virtual consoles (VCs). You can switch between "
+            "these using the F1-F11 keys.\n\n"
+
+            "This is but a small glimpse of what you can do with HelenOS. "
             "To learn more please point your browser to the HelenOS User's "
-            "Guide: http://trac.helenos.org/trac.fcgi/wiki/UsersGuide\n\n");
+            "Guide: http://trac.helenos.org/trac.fcgi/wiki/UsersGuide\n\n",
+            ALIGN_LEFT);
 }
 

uspace/app/bdsh/cmds/modules/ls/ls.c

@@ -169 +169 @@
                 
                 /* fill the name field */
-                tosort[nbdirs].name = (char *) malloc(str_length(dp->d_name) + 1);
+                tosort[nbdirs].name = (char *) malloc(str_size(dp->d_name) + 1);
                 if (!tosort[nbdirs].name) {
                         cli_error(CL_ENOMEM, "ls: failed to scan %s", d);
                         goto out;
                 }
-                
-                str_cpy(tosort[nbdirs].name, str_length(dp->d_name) + 1, dp->d_name);
+
+                str_cpy(tosort[nbdirs].name, str_size(dp->d_name) + 1, dp->d_name);
                 len = snprintf(buff, PATH_MAX - 1, "%s/%s", d, tosort[nbdirs].name);
                 buff[len] = '\0';

uspace/app/bdsh/config.h

@@ -40 +40 @@
 #endif
 
-/* Work around for getenv() */
-#define PATH "/srv:/app"
-#define PATH_DELIM ":"
-
 /* Used in many places */
 #define SMALL_BUFLEN 256

uspace/app/bdsh/exec.c

@@ -40 +40 @@
 #include <str_error.h>
 #include <errno.h>
+#include <vfs/vfs.h>
 
 #include "config.h"
@@ -51 +52 @@
 static char *find_command(char *);
 static int try_access(const char *);
+
+const char *search_dir[] = { "/app", "/srv", NULL };
 
 /* work-around for access() */
@@ -69 +72 @@
 static char *find_command(char *cmd)
 {
-        char *path_tok;
-        char *path[PATH_MAX];
-        int n = 0, i = 0;
-        size_t x = str_size(cmd) + 2;
+        size_t i;
 
         found = (char *)malloc(PATH_MAX);
@@ -81 +81 @@
         }
 
-        path_tok = str_dup(PATH);
-
-        /* Extract the PATH env to a path[] array */
-        path[n] = strtok(path_tok, PATH_DELIM);
-        while (NULL != path[n]) {
-                if ((str_size(path[n]) + x ) > PATH_MAX) {
-                        cli_error(CL_ENOTSUP,
-                                "Segment %d of path is too large, search ends at segment %d",
-                                n, n-1);
-                        break;
-                }
-                path[++n] = strtok(NULL, PATH_DELIM);
-        }
-
         /* We now have n places to look for the command */
-        for (i=0; path[i]; i++) {
+        for (i = 0; search_dir[i] != NULL; i++) {
                 memset(found, 0, sizeof(found));
-                snprintf(found, PATH_MAX, "%s/%s", path[i], cmd);
+                snprintf(found, PATH_MAX, "%s/%s", search_dir[i], cmd);
                 if (-1 != try_access(found)) {
-                        free(path_tok);
                         return (char *) found;
                 }
@@ -106 +91 @@
 
         /* We didn't find it, just give it back as-is. */
-        free(path_tok);
         return (char *) cmd;
 }
@@ -116 +100 @@
         char *tmp;
         int rc, retval, i;
-        fdi_node_t file_nodes[3];
-        fdi_node_t *file_nodes_p[4];
+        int file_handles[3];
+        int *file_handles_p[4];
         FILE *files[3];
 
@@ -128 +112 @@
         
         for (i = 0; i < 3 && files[i] != NULL; i++) {
-                if (fnode(files[i], &file_nodes[i]) == EOK) {
-                        file_nodes_p[i] = &file_nodes[i];
+                if (fhandle(files[i], &file_handles[i]) == EOK) {
+                        file_handles_p[i] = &file_handles[i];
                 }
                 else {
-                        file_nodes_p[i] = NULL;
+                        file_handles_p[i] = NULL;
                 }
         }
-        file_nodes_p[i] = NULL;
+        file_handles_p[i] = NULL;
 
-        rc = task_spawnvf(&tid, tmp, (const char **) argv, file_nodes_p);
+        rc = task_spawnvf(&tid, tmp, (const char **) argv, file_handles_p);
         free(tmp);
 

uspace/app/bdsh/exec.h

@@ -33 +33 @@
 #include "scli.h"
 
+extern const char *search_dir[];
+
 extern unsigned int try_exec(char *, char **, iostate_t *);
 

uspace/app/bdsh/input.c

@@ -1 +1 @@
 /*
  * Copyright (c) 2008 Tim Post
+ * Copyright (c) 2011 Jiri Svoboda
+ * Copyright (c) 2011 Martin Sucha
  * All rights reserved.
  *
@@ -43 +45 @@
 
 #include "config.h"
+#include "compl.h"
 #include "util.h"
 #include "scli.h"
@@ -65 +68 @@
 {
         char *cmd[WORD_MAX];
+        token_t *tokens = calloc(WORD_MAX, sizeof(token_t));
+        if (tokens == NULL)
+                return ENOMEM;
         int rc = 0;
         tokenizer_t tok;
-        int i, pipe_count, processed_pipes;
-        int pipe_pos[2];
-        char **actual_cmd;
+        unsigned int i, pipe_count, processed_pipes;
+        unsigned int pipe_pos[2];
         char *redir_from = NULL;
         char *redir_to = NULL;
 
-        if (NULL == usr->line)
+        if (NULL == usr->line) {
+                free(tokens);
                 return CL_EFAIL;
-
-        rc = tok_init(&tok, usr->line, cmd, WORD_MAX);
+        }
+
+        rc = tok_init(&tok, usr->line, tokens, WORD_MAX);
         if (rc != EOK) {
                 goto finit;
         }
         
-        rc = tok_tokenize(&tok);
+        size_t tokens_length;
+        rc = tok_tokenize(&tok, &tokens_length);
         if (rc != EOK) {
                 goto finit;
+        }
+        
+        if (tokens_length > 0 && tokens[0].type == TOKTYPE_SPACE) {
+                tokens++;
+                tokens_length--;
+        }
+        
+        if (tokens_length > 0 && tokens[tokens_length-1].type == TOKTYPE_SPACE) {
+                tokens_length--;
         }
         
@@ -91 +108 @@
          * First find the pipes and check that there are no more
          */
-        int cmd_length = 0;
-        for (i = 0, pipe_count = 0; cmd[i] != NULL; i++, cmd_length++) {
-                if (cmd[i][0] == '|') {
+        for (i = 0, pipe_count = 0; i < tokens_length; i++) {
+                if (tokens[i].type == TOKTYPE_PIPE) {
                         if (pipe_count >= 2) {
                                 print_pipe_usage();
@@ -104 +120 @@
         }
         
-        actual_cmd = cmd;
+        unsigned int cmd_token_start = 0;
+        unsigned int cmd_token_end = tokens_length;
+        
         processed_pipes = 0;
         
         /* Check if the first part (from <file> |) is present */
-        if (pipe_count > 0 && pipe_pos[0] == 2 && str_cmp(cmd[0], "from") == 0) {
+        if (pipe_count > 0 && (pipe_pos[0] == 3 || pipe_pos[0] == 4) && str_cmp(tokens[0].text, "from") == 0) {
                 /* Ignore the first three tokens (from, file, pipe) and set from */
-                redir_from = cmd[1];
-                actual_cmd = cmd + 3;
+                redir_from = tokens[2].text;
+                cmd_token_start = pipe_pos[0]+1;
                 processed_pipes++;
         }
@@ -117 +135 @@
         /* Check if the second part (| to <file>) is present */
         if ((pipe_count - processed_pipes) > 0 &&
-            pipe_pos[processed_pipes] == cmd_length - 3 &&
-            str_cmp(cmd[cmd_length-2], "to") == 0) {
+            (pipe_pos[processed_pipes] == tokens_length - 4 ||
+            (pipe_pos[processed_pipes] == tokens_length - 5 &&
+            tokens[tokens_length-4].type == TOKTYPE_SPACE )) &&
+            str_cmp(tokens[tokens_length-3].text, "to") == 0) {
                 /* Ignore the last three tokens (pipe, to, file) and set to */
-                redir_to = cmd[cmd_length-1];
-                cmd[cmd_length-3] = NULL;
-                cmd_length -= 3;
+                redir_to = tokens[tokens_length-1].text;
+                cmd_token_end = pipe_pos[processed_pipes];
                 processed_pipes++;
         }
@@ -132 +151 @@
         }
         
-        if (actual_cmd[0] == NULL) {
+        /* Convert tokens of the command to string array */
+        unsigned int cmd_pos = 0;
+        for (i = cmd_token_start; i < cmd_token_end; i++) {
+                if (tokens[i].type != TOKTYPE_SPACE) {
+                        cmd[cmd_pos++] = tokens[i].text;
+                }
+        }
+        cmd[cmd_pos++] = NULL;
+        
+        if (cmd[0] == NULL) {
                 print_pipe_usage();
                 rc = ENOTSUP;
@@ -184 +212 @@
         }
         tok_fini(&tok);
+        free(tokens);
 
         return rc;
@@ -226 +255 @@
         int rc;
         
-        console_flush(tinput->console);
-        console_set_style(tinput->console, STYLE_EMPHASIS);
-        printf("%s", usr->prompt);
-        console_flush(tinput->console);
-        console_set_style(tinput->console, STYLE_NORMAL);
+        tinput_set_prompt(tinput, usr->prompt);
 
         rc = tinput_read(tinput, &str);
@@ -263 +288 @@
         }
 
+        tinput_set_compl_ops(tinput, &compl_ops);
+
         return 0;
 }

uspace/app/bdsh/scli.h

@@ -32 +32 @@
 #include "config.h"
 #include <stdint.h>
+#include <stdio.h>
 
 typedef struct {

uspace/app/bdsh/tok.c

@@ -42 +42 @@
 static bool tok_pending_chars(tokenizer_t *);
 static int tok_finish_string(tokenizer_t *);
+static void tok_start_token(tokenizer_t *, token_type_t);
 
 /** Initialize the token parser
@@ -50 +51 @@
  * @param max_tokens number of elements of the out_tokens array
  */
-int tok_init(tokenizer_t *tok, char *input, char **out_tokens,
+int tok_init(tokenizer_t *tok, char *input, token_t *out_tokens,
     size_t max_tokens)
 {       
         tok->in = input;
         tok->in_offset = 0;
+        tok->last_in_offset = 0;
+        tok->in_char_offset = 0;
+        tok->last_in_char_offset = 0;
         
         tok->outtok = out_tokens;
         tok->outtok_offset = 0;
-        /* Leave one slot for a null terminator */
-        assert(max_tokens > 0);
-        tok->outtok_size = max_tokens - 1;
+        tok->outtok_size = max_tokens;
         
         /* Prepare a buffer where all the token strings will be stored */
@@ -87 +89 @@
 
 /** Tokenize the input string into the tokens */
-int tok_tokenize(tokenizer_t *tok)
+int tok_tokenize(tokenizer_t *tok, size_t *tokens_length)
 {
         int rc;
-        wchar_t cur_char;
+        wchar_t next_char;
         
         /* Read the input line char by char and append tokens */
-        while ((cur_char = tok_get_char(tok)) != 0) {
-                if (cur_char == ' ') {
-                        /* Spaces delimit tokens, but are not processed in any way
-                         * Push the token if there is any.
+        while ((next_char = tok_look_char(tok)) != 0) {
+                if (next_char == ' ') {
+                        /* Push the token if there is any.
                          * There may not be any pending char for a token in case
                          * there are several spaces in the input.
@@ -106 +107 @@
                                 }
                         }
-                }
-                else if (cur_char == '|') {
-                        /* Pipes are tokens that are delimiters and should be output
-                         * as a separate token
+                        tok_start_token(tok, TOKTYPE_SPACE);
+                        /* Eat all the spaces */
+                        while (tok_look_char(tok) == ' ') {
+                                tok_push_char(tok, tok_get_char(tok));
+                        }
+                        tok_push_token(tok);
+                        
+                }
+                else if (next_char == '|') {
+                        /* Pipes are tokens that are delimiters and should be
+                         * output as a separate token
                          */
                         if (tok_pending_chars(tok)) {
@@ -118 +126 @@
                         }
                         
-                        rc = tok_push_char(tok, '|');
+                        tok_start_token(tok, TOKTYPE_PIPE);
+                        
+                        rc = tok_push_char(tok, tok_get_char(tok));
                         if (rc != EOK) {
                                 return rc;
@@ -128 +138 @@
                         }
                 }
-                else if (cur_char == '\'') {
+                else if (next_char == '\'') {
                         /* A string starts with a quote (') and ends again with a quote.
                          * A literal quote is written as ''
                          */
+                        tok_start_token(tok, TOKTYPE_TEXT);
+                        /* Eat the quote */
+                        tok_get_char(tok);
                         rc = tok_finish_string(tok);
                         if (rc != EOK) {
@@ -138 +151 @@
                 }
                 else {
+                        if (!tok_pending_chars(tok)) {
+                                tok_start_token(tok, TOKTYPE_TEXT);
+                        }
                         /* If we are handling any other character, just append it to
                          * the current token.
                          */
-                        rc = tok_push_char(tok, cur_char);
+                        rc = tok_push_char(tok, tok_get_char(tok));
                         if (rc != EOK) {
                                 return rc;
@@ -156 +172 @@
         }
         
-        /* We always have a space for the terminator, as we
-         * reserved it in tok_init */
-        tok->outtok[tok->outtok_offset] = 0;
+        *tokens_length = tok->outtok_offset;
         
         return EOK;
@@ -167 +181 @@
 {
         int rc;
-        wchar_t cur_char;
-        
-        while ((cur_char = tok_get_char(tok)) != 0) {
-                if (cur_char == '\'') {
+        wchar_t next_char;
+        
+        while ((next_char = tok_look_char(tok)) != 0) {
+                if (next_char == '\'') {
+                        /* Eat the quote */
+                        tok_get_char(tok);
                         if (tok_look_char(tok) == '\'') {
                                 /* Encode a single literal quote */
@@ -187 +203 @@
                 }
                 else {
-                        rc = tok_push_char(tok, cur_char);
+                        rc = tok_push_char(tok, tok_get_char(tok));
                         if (rc != EOK) {
                                 return rc;
@@ -201 +217 @@
 wchar_t tok_get_char(tokenizer_t *tok)
 {
+        tok->in_char_offset++;
         return str_decode(tok->in, &tok->in_offset, STR_NO_LIMIT);
 }
@@ -208 +225 @@
 {
         size_t old_offset = tok->in_offset;
+        size_t old_char_offset = tok->in_char_offset;
         wchar_t ret = tok_get_char(tok);
         tok->in_offset = old_offset;
+        tok->in_char_offset = old_char_offset;
         return ret;
 }
@@ -219 +238 @@
 }
 
+void tok_start_token(tokenizer_t *tok, token_type_t type)
+{
+        tok->current_type = type;
+}
+
 /** Push the current token to the output array */
 int tok_push_token(tokenizer_t *tok)
@@ -231 +255 @@
         
         tok->outbuf[tok->outbuf_offset++] = 0;
-        tok->outtok[tok->outtok_offset++] = tok->outbuf + tok->outbuf_last_start;
+        token_t *tokinfo = &tok->outtok[tok->outtok_offset++];
+        tokinfo->type = tok->current_type;
+        tokinfo->text = tok->outbuf + tok->outbuf_last_start;
+        tokinfo->byte_start = tok->last_in_offset;
+        tokinfo->byte_length = tok->in_offset - tok->last_in_offset;
+        tokinfo->char_start = tok->last_in_char_offset;
+        tokinfo->char_length = tok->in_char_offset - tok->last_in_char_offset;
         tok->outbuf_last_start = tok->outbuf_offset;
+        
+        /* We have consumed the first char of the next token already */
+        tok->last_in_offset = tok->in_offset;
+        tok->last_in_char_offset = tok->in_char_offset;
         
         return EOK;

uspace/app/bdsh/tok.h

@@ -30 +30 @@
 #define TOK_H
 
+typedef enum {
+        TOKTYPE_TEXT,
+        TOKTYPE_PIPE,
+        TOKTYPE_SPACE
+} token_type_t;
+
+typedef struct {
+        char *text;
+        size_t byte_start;
+        size_t char_start;
+        size_t byte_length;
+        size_t char_length;
+        token_type_t type;
+} token_t;
+
 typedef struct {
         char *in;
         size_t in_offset;
+        size_t last_in_offset;
+        size_t in_char_offset;
+        size_t last_in_char_offset;
         
         char *outbuf;
@@ -39 +57 @@
         size_t outbuf_last_start;
         
-        char **outtok;
+        token_t *outtok;
+        token_type_t current_type;
         size_t outtok_offset;
         size_t outtok_size;
 } tokenizer_t;
 
-extern int tok_init(tokenizer_t *, char *, char **, size_t);
+extern int tok_init(tokenizer_t *, char *, token_t *, size_t);
 extern void tok_fini(tokenizer_t *);
-extern int tok_tokenize(tokenizer_t *);
+extern int tok_tokenize(tokenizer_t *, size_t *);
 
 #endif