--- coreutils-8.24/src/cut.c 2015-06-26 19:05:22.000000000 +0200 +++ cut.c 2016-01-15 10:15:04.863804121 +0100 @@ -28,6 +28,11 @@ #include #include #include + +#include +#include +#include + #include "system.h" #include "error.h" @@ -61,25 +66,16 @@ CURRENT_RP.HI then we make CURRENT_RP to point to the next range pair. */ static struct field_range_pair *current_rp; -/* This buffer is used to support the semantics of the -s option - (or lack of same) when the specified field list includes (does - not include) the first field. In both of those cases, the entire - first field must be read into this buffer to determine whether it - is followed by a delimiter or a newline before any of it may be - output. Otherwise, cut_fields can do the job without using this - buffer. */ -static char *field_1_buffer; - -/* The number of bytes allocated for FIELD_1_BUFFER. */ -static size_t field_1_bufsize; - enum operating_mode { undefined_mode, - /* Output characters that are in the given bytes. */ + /* Output the given bytes. */ byte_mode, + /* Output characters that are in the given positions . */ + char_mode, + /* Output the given delimiter-separated fields. */ field_mode }; @@ -91,12 +87,16 @@ static enum operating_mode operating_mode; with field mode. */ static bool suppress_non_delimited; +/* Unless true, we do not recognize multibyte characters in byte-splitting + mode. */ +static bool no_break_mb_chars; + /* If true, print all bytes, characters, or fields _except_ those that were specified. */ static bool complement; /* The delimiter character for field mode. */ -static unsigned char delim; +static mbf_char_t delim; /* The delimiter for each line/record. */ static unsigned char line_delim = '\n'; @@ -109,7 +109,7 @@ static size_t output_delimiter_length; /* The output field separator string. Defaults to the 1-character string consisting of the input delimiter. */ -static char *output_delimiter_string; +static char const *output_delimiter_string; /* True if we have ever read standard input. */ static bool have_read_stdin; @@ -164,7 +164,7 @@ Print selected parts of lines from each FILE to standard output.\n\ -f, --fields=LIST select only these fields; also print any line\n\ that contains no delimiter character, unless\n\ the -s option is specified\n\ - -n (ignored)\n\ + -n with -b, don't split multibyte characters\n\ "), stdout); fputs (_("\ --complement complement the set of selected bytes, characters\n\ @@ -211,6 +211,12 @@ next_item (size_t *item_idx) current_rp++; } +static inline void +next_item_n (size_t *item_idx, size_t n) +{ + while (n-- > 0) + next_item (item_idx); +} /* Return nonzero if the K'th field or byte is printable. */ static inline bool @@ -219,6 +225,15 @@ print_kth (size_t k) return current_rp->lo <= k; } +/* The lo and hi params should be used for the current characters byte position + * and byte size, respectively. */ +static inline bool +rp_intersect (size_t lo, size_t hi) +{ + return ((current_rp->lo <= lo && current_rp->hi >= lo) + || (current_rp->lo <= hi && current_rp->hi >= hi)); +} + /* Return nonzero if K'th byte is the beginning of a range. */ static inline bool @@ -281,23 +296,215 @@ cut_bytes (FILE *stream) } /* Read from stream STREAM, printing to standard output any selected fields. */ +extern ssize_t +mb_getndelim2 (mbf_char_t **lineptr, size_t *linesize, size_t nmax, + mbf_char_t delim1, mbf_char_t delim2, mb_file_t *stream) +{ +/* The maximum value that getndelim2 can return without suffering from + overflow problems, either internally (because of pointer + subtraction overflow) or due to the API (because of ssize_t). */ +#define GETNDELIM2_MAXIMUM (PTRDIFF_MAX < SSIZE_MAX ? PTRDIFF_MAX : SSIZE_MAX) + +/* Try to add at least this many bytes when extending the buffer. + MIN_CHUNK must be no greater than GETNDELIM2_MAXIMUM. */ +#define MIN_CHUNK 64 + size_t nchars_avail; /* Allocated but unused chars in *LINEPTR. */ + mbf_char_t *read_pos; /* Where we're reading into *LINEPTR. */ + ssize_t chars_stored = -1; + mbf_char_t *ptr = *lineptr; + size_t size = *linesize; + bool found_delimiter; + + if (!ptr) + { + size = nmax < MIN_CHUNK ? nmax : MIN_CHUNK; + ptr = malloc (size * sizeof (mbf_char_t)); + if (!ptr) + return -1; + } + + if (size < 0) + goto done; + + nchars_avail = size; + read_pos = ptr; + + if (nchars_avail == 0 && nmax <= size) + goto done; + + /* Normalize delimiters, since memchr2 doesn't handle EOF. */ + if (mb_iseof (delim1)) + mb_copy (&delim1, &delim2); + else if (mb_iseof (delim2)) + mb_copy (&delim2, &delim1); + + flockfile (stream); + + found_delimiter = false; + do + { + /* Here always ptr + size == read_pos + nchars_avail. + Also nchars_avail > 0 || size < nmax. */ + + mbf_char_t c IF_LINT (= 0); + { + mbf_getc (c, *stream); + if (mb_iseof (c)) + { + /* Return partial line, if any. */ + if (read_pos == ptr) + goto unlock_done; + else + break; + } + if (mb_equal (c, delim1) || mb_equal (c, delim2)) + found_delimiter = true; + } + + /* We always want at least one byte left in the buffer, since we + always (unless we get an error while reading the first byte) + NUL-terminate the line buffer. */ + + if (!nchars_avail) + { + /* Grow size proportionally, not linearly, to avoid O(n^2) + running time. */ + size_t newsize = size < MIN_CHUNK ? size + MIN_CHUNK : 2 * size; + mbf_char_t *newptr; + + /* Respect nmax. This handles possible integer overflow. */ + if (! (size < newsize && newsize <= nmax)) + newsize = nmax; + + if (GETNDELIM2_MAXIMUM < newsize) + { + size_t newsizemax = GETNDELIM2_MAXIMUM + 1; + if (size == newsizemax) + goto unlock_done; + newsize = newsizemax; + } + nchars_avail = newsize - (read_pos - ptr); + newptr = realloc (ptr, newsize * sizeof (mbf_char_t)); + if (!newptr) + goto unlock_done; + ptr = newptr; + size = newsize; + read_pos = size - nchars_avail + ptr; + } + + /* Here, if size < nmax, nchars_avail >= buffer_len + 1. + If size == nmax, nchars_avail > 0. */ + + if (1 < nchars_avail--) + { + mb_copy(read_pos++, &c); + } + + } + while (!found_delimiter); + + chars_stored = (read_pos - ptr); + + unlock_done: + funlockfile (stream); + + done: + *lineptr = ptr; + *linesize = size; + return chars_stored; +} + +static void +cut_chars (FILE *stream) +{ + size_t char_idx; /* Number of chars in the line so far. */ + bool print_delimiter; + mbf_char_t c; + mb_file_t mbf; + + print_delimiter = false; + char_idx = 0; + current_rp = frp; + + mbf_init (mbf, stream); + while (true) + { + mbf_getc (c, mbf); + + if (mb_iseq (c, line_delim)) + { + putc (line_delim, stdout); + char_idx = 0; + print_delimiter = false; + current_rp = frp; + } + else if (mb_iseof (c)) + { + if (char_idx > 0) + putc (line_delim, stdout); + break; + } + else + { + /* Forward by one byte. */ + next_item (&char_idx); + + /* Check if the current characters byte range is within + * the argument list. */ + if (rp_intersect (char_idx, char_idx + mb_len (c) - 1)) + { + if (output_delimiter_specified) + { + if (print_delimiter && is_range_start_index (char_idx)) + { + fwrite (output_delimiter_string, sizeof (char), + output_delimiter_length, stdout); + } + print_delimiter = true; + } + mb_putc (c, stdout); + } + + /* Byte mode with multibyte characters uncut (-b -n). */ + if (no_break_mb_chars) + /* Forward by an additional byte_length (c) - 1. */ + next_item_n (&char_idx, mb_len (c) - 1); + } + } +} static void cut_fields (FILE *stream) { - int c; + + /* This buffer is used to support the semantics of the -s option + (or lack of same) when the specified field list includes (does + not include) the first field. In both of those cases, the entire + first field must be read into this buffer to determine whether it + is followed by a delimiter or a newline before any of it may be + output. Otherwise, cut_fields can do the job without using this + buffer. */ + mbf_char_t *field_1_buffer = 0; + /* The number of bytes allocated for FIELD_1_BUFFER. */ + size_t field_1_bufsize; + + + mbf_char_t c, d; + mb_file_t mbf; size_t field_idx = 1; bool found_any_selected_field = false; bool buffer_first_field; current_rp = frp; - c = getc (stream); - if (c == EOF) + mbf_init (mbf, stream); + mbf_getc (c, mbf); + if (mb_iseof (c)) return; - ungetc (c, stream); - c = 0; + mbf_ungetc (c, mbf); + mb_setascii (&c, 0); + mb_copy (&d, &delim); /* To support the semantics of the -s flag, we may have to buffer all of the first field to determine whether it is 'delimited.' @@ -312,10 +519,14 @@ cut_fields (FILE *stream) if (field_idx == 1 && buffer_first_field) { ssize_t len; - size_t n_bytes; + size_t n_chars; + mbf_char_t nl; + mb_setascii (&nl, line_delim); + + len = mb_getndelim2 (&field_1_buffer, &field_1_bufsize, + GETNLINE_NO_LIMIT, d, nl, &mbf); + - len = getndelim2 (&field_1_buffer, &field_1_bufsize, 0, - GETNLINE_NO_LIMIT, delim, line_delim, stream); if (len < 0) { free (field_1_buffer); @@ -325,15 +536,15 @@ cut_fields (FILE *stream) xalloc_die (); } - n_bytes = len; - assert (n_bytes != 0); + n_chars = len; + //assert (n_chars != 0); - c = 0; + mb_setascii (&c, 0); /* If the first field extends to the end of line (it is not delimited) and we are printing all non-delimited lines, print this one. */ - if (to_uchar (field_1_buffer[n_bytes - 1]) != delim) + if (!mb_equal (field_1_buffer[n_chars - 1], d)) { if (suppress_non_delimited) { @@ -341,26 +552,30 @@ cut_fields (FILE *stream) } else { - fwrite (field_1_buffer, sizeof (char), n_bytes, stdout); + for (int i = 0; i < n_chars; ++i) + mb_putc (field_1_buffer[i], stdout); + /* Make sure the output line is newline terminated. */ - if (field_1_buffer[n_bytes - 1] != line_delim) + if (!mb_iseq (field_1_buffer[n_chars - 1], line_delim)) putchar (line_delim); - c = line_delim; + mb_setascii (&c, line_delim); } continue; } if (print_kth (1)) { /* Print the field, but not the trailing delimiter. */ - fwrite (field_1_buffer, sizeof (char), n_bytes - 1, stdout); + for (int i = 0; i < n_chars - 1; ++i) + mb_putc (field_1_buffer[i], stdout); /* With -d$'\n' don't treat the last '\n' as a delimiter. */ - if (delim == line_delim) + if (mb_iseq (d, line_delim)) { - int last_c = getc (stream); - if (last_c != EOF) + mbf_char_t last_c; + mbf_getc (last_c, mbf); + if (!mb_iseof (last_c)) { - ungetc (last_c, stream); + mbf_ungetc (last_c, mbf); found_any_selected_field = true; } } @@ -370,7 +585,8 @@ cut_fields (FILE *stream) next_item (&field_idx); } - int prev_c = c; + mbf_char_t prev_c; + mb_copy (&prev_c, &c); if (print_kth (field_idx)) { @@ -381,42 +597,46 @@ cut_fields (FILE *stream) } found_any_selected_field = true; - while ((c = getc (stream)) != delim && c != line_delim && c != EOF) + mbf_getc (c, mbf); + while (!mb_equal (c, d) && !mb_iseq (c, line_delim) && !mb_iseof (c)) { - putchar (c); - prev_c = c; + mb_putc (c, stdout); + mb_copy (&prev_c, &c); + mbf_getc (c, mbf); } } else { - while ((c = getc (stream)) != delim && c != line_delim && c != EOF) + mbf_getc (c, mbf); + while (!mb_equal (c, d) && !mb_iseq (c, line_delim) && !mb_iseof (c)) { - prev_c = c; + mb_copy (&prev_c, &c); + mbf_getc (c, mbf); } } /* With -d$'\n' don't treat the last '\n' as a delimiter. */ - if (delim == line_delim && c == delim) + if (mb_iseq (d, line_delim) && mb_equal (c, d)) { - int last_c = getc (stream); - if (last_c != EOF) - ungetc (last_c, stream); + mbf_char_t last_c; + mbf_getc (last_c, mbf); + if (!mb_iseof (last_c)) + mbf_ungetc (last_c, mbf); else - c = last_c; + mb_copy (&c, &last_c); } - if (c == delim) + if (mb_equal (c, d)) next_item (&field_idx); - else if (c == line_delim || c == EOF) + else if (mb_iseq (c, line_delim) || mb_iseof (c)) { if (found_any_selected_field || !(suppress_non_delimited && field_idx == 1)) { - if (c == line_delim || prev_c != line_delim - || delim == line_delim) + if (mb_iseq (c, line_delim) || !mb_iseq (prev_c, line_delim) || mb_iseq (d, line_delim)) putchar (line_delim); } - if (c == EOF) + if (mb_iseof (c)) break; field_idx = 1; current_rp = frp; @@ -429,7 +649,14 @@ static void cut_stream (FILE *stream) { if (operating_mode == byte_mode) - cut_bytes (stream); + { + if (no_break_mb_chars) + cut_chars (stream); + else + cut_bytes (stream); + } + else if (operating_mode == char_mode) + cut_chars (stream); else cut_fields (stream); } @@ -483,6 +710,7 @@ main (int argc, char **argv) bool ok; bool delim_specified = false; char *spec_list_string IF_LINT ( = NULL); + mbi_iterator_t iter; initialize_main (&argc, &argv); set_program_name (argv[0]); @@ -496,8 +724,10 @@ main (int argc, char **argv) /* By default, all non-delimited lines are printed. */ suppress_non_delimited = false; + /* Default behaviour for -b, unless -n is also specified. */ + no_break_mb_chars = false; - delim = '\0'; + mb_setascii (&delim, '\0'); have_read_stdin = false; while ((optc = getopt_long (argc, argv, "b:c:d:f:nsz", longopts, NULL)) != -1) @@ -505,7 +735,6 @@ main (int argc, char **argv) switch (optc) { case 'b': - case 'c': /* Build the byte list. */ if (operating_mode != undefined_mode) FATAL_ERROR (_("only one type of list may be specified")); @@ -513,6 +742,14 @@ main (int argc, char **argv) spec_list_string = optarg; break; + case 'c': + /* Build the char list. */ + if (operating_mode != undefined_mode) + FATAL_ERROR (_("only one type of list may be specified")); + operating_mode = char_mode; + spec_list_string = optarg; + break; + case 'f': /* Build the field list. */ if (operating_mode != undefined_mode) @@ -524,9 +761,17 @@ main (int argc, char **argv) case 'd': /* New delimiter. */ /* Interpret -d '' to mean 'use the NUL byte as the delimiter.' */ - if (optarg[0] != '\0' && optarg[1] != '\0') + mbi_init (iter, optarg, strlen (optarg)); + if (!mbi_avail (iter)) + mb_setascii (&delim, '\0'); + else + { + mb_copy (&delim, &mbi_cur (iter)); + + mbi_advance (iter); + if (mbi_avail (iter)) FATAL_ERROR (_("the delimiter must be a single character")); - delim = optarg[0]; + } delim_specified = true; break; @@ -540,6 +785,7 @@ main (int argc, char **argv) break; case 'n': + no_break_mb_chars = true; break; case 's': @@ -579,15 +825,12 @@ main (int argc, char **argv) | (complement ? SETFLD_COMPLEMENT : 0) ); if (!delim_specified) - delim = '\t'; + mb_setascii (&delim, '\t'); if (output_delimiter_string == NULL) { - static char dummy[2]; - dummy[0] = delim; - dummy[1] = '\0'; - output_delimiter_string = dummy; - output_delimiter_length = 1; + output_delimiter_string = mb_ptr (delim); + output_delimiter_length = mb_len (delim); } if (optind == argc)