From f14db65234364c75d1efbad1f1a7d1e32ed4686a Mon Sep 17 00:00:00 2001 From: Tim Waugh Date: Mon, 15 May 2006 14:10:12 +0000 Subject: [PATCH] Fixed i18n patch. --- coreutils-i18n.patch | 3737 +++++++++++++++++++++--------------------- 1 file changed, 1869 insertions(+), 1868 deletions(-) diff --git a/coreutils-i18n.patch b/coreutils-i18n.patch index 481f8bb..d841385 100644 --- a/coreutils-i18n.patch +++ b/coreutils-i18n.patch @@ -1,49 +1,23 @@ ---- coreutils-5.93/lib/linebuffer.h.i18n 2005-05-14 08:58:06.000000000 +0100 -+++ coreutils-5.93/lib/linebuffer.h 2005-12-23 08:53:01.000000000 +0000 -@@ -22,6 +22,11 @@ - - # include - -+/* Get mbstate_t. */ -+# if HAVE_WCHAR_H -+# include -+# endif -+ - /* A `struct linebuffer' holds a line of text. */ - - struct linebuffer -@@ -29,6 +34,9 @@ - size_t size; /* Allocated. */ - size_t length; /* Used. */ - char *buffer; -+# if HAVE_WCHAR_H -+ mbstate_t state; -+# endif - }; - - /* Initialize linebuffer LINEBUFFER for use. */ ---- coreutils-5.93/src/cut.c.i18n 2005-08-12 08:16:25.000000000 +0100 -+++ coreutils-5.93/src/cut.c 2005-12-23 08:53:01.000000000 +0000 -@@ -29,6 +29,11 @@ - #include +--- coreutils-5.95/src/expand.c.i18n 2005-08-12 08:16:25.000000000 +0100 ++++ coreutils-5.95/src/expand.c 2006-05-15 15:08:57.000000000 +0100 +@@ -38,11 +38,28 @@ + #include #include #include + -+/* Get mbstate_t, mbrtowc(). */ ++/* Get mbstate_t, mbrtowc(), wcwidth(). */ +#if HAVE_WCHAR_H +# include +#endif ++ #include "system.h" - #include "error.h" -@@ -37,6 +42,18 @@ #include "quote.h" #include "xstrndup.h" +/* MB_LEN_MAX is incorrectly defined to be 1 in at least one GCC -+ installation; work around this configuration error. */ ++ installation; work around this configuration error. */ +#if !defined MB_LEN_MAX || MB_LEN_MAX < 2 -+# undef MB_LEN_MAX +# define MB_LEN_MAX 16 +#endif + @@ -53,570 +27,175 @@ +#endif + /* The official name of this program (e.g., no `g' prefix). */ - #define PROGRAM_NAME "cut" + #define PROGRAM_NAME "expand" -@@ -67,6 +84,52 @@ - } \ - while (0) - -+/* Refill the buffer BUF to get a multibyte character. */ -+#define REFILL_BUFFER(BUF, BUFPOS, BUFLEN, STREAM) \ -+ do \ -+ { \ -+ if (BUFLEN < MB_LEN_MAX && !feof (STREAM) && !ferror (STREAM)) \ -+ { \ -+ memmove (BUF, BUFPOS, BUFLEN); \ -+ BUFLEN += fread (BUF + BUFLEN, sizeof(char), BUFSIZ, STREAM); \ -+ BUFPOS = BUF; \ -+ } \ -+ } \ -+ while (0) -+ -+/* Get wide character on BUFPOS. BUFPOS is not included after that. -+ If byte sequence is not valid as a character, CONVFAIL is 1. Otherwise 0. */ -+#define GET_NEXT_WC_FROM_BUFFER(WC, BUFPOS, BUFLEN, MBLENGTH, STATE, CONVFAIL) \ -+ do \ -+ { \ -+ mbstate_t state_bak; \ -+ \ -+ if (BUFLEN < 1) \ -+ { \ -+ WC = WEOF; \ -+ break; \ -+ } \ -+ \ -+ /* Get a wide character. */ \ -+ CONVFAIL = 0; \ -+ state_bak = STATE; \ -+ MBLENGTH = mbrtowc ((wchar_t *)&WC, BUFPOS, BUFLEN, &STATE); \ -+ \ -+ switch (MBLENGTH) \ -+ { \ -+ case (size_t)-1: \ -+ case (size_t)-2: \ -+ CONVFAIL++; \ -+ STATE = state_bak; \ -+ /* Fall througn. */ \ -+ \ -+ case 0: \ -+ MBLENGTH = 1; \ -+ break; \ -+ } \ -+ } \ -+ while (0) -+ - struct range_pair - { - size_t lo; -@@ -85,7 +148,7 @@ - /* The number of bytes allocated for FIELD_1_BUFFER. */ - static size_t field_1_bufsize; - --/* The largest field or byte index used as an endpoint of a closed -+/* The largest byte, character or field index used as an endpoint of a closed - or degenerate range specification; this doesn't include the starting - index of right-open-ended ranges. For example, with either range spec - `2-5,9-', `2-3,5,9-' this variable would be set to 5. */ -@@ -97,10 +160,11 @@ - - /* This is a bit vector. - In byte mode, which bytes to output. -+ In character mode, which characters to output. - In field mode, which DELIM-separated fields to output. -- Both bytes and fields are numbered starting with 1, -+ Bytes, characters and fields are numbered starting with 1, - so the zeroth bit of this array is unused. -- A field or byte K has been selected if -+ A byte, character or field K has been selected if - (K <= MAX_RANGE_ENDPOINT and is_printable_field(K)) - || (EOL_RANGE_START > 0 && K >= EOL_RANGE_START). */ - static unsigned char *printable_field; -@@ -109,9 +173,12 @@ - { - undefined_mode, - -- /* Output characters that are in the given bytes. */ -+ /* Output bytes that are at the given positions. */ - byte_mode, - -+ /* Output characters that are at the given positions. */ -+ character_mode, -+ - /* Output the given delimeter-separated fields. */ - field_mode - }; -@@ -121,6 +188,13 @@ - - static enum operating_mode operating_mode; - -+/* If nonzero, when in byte mode, don't split multibyte characters. */ -+static int byte_mode_character_aware; -+ -+/* If nonzero, the function for single byte locale is work -+ if this program runs on multibyte locale. */ -+static int force_singlebyte_mode; -+ - /* If true do not output lines containing no delimeter characters. - Otherwise, all such lines are printed. This option is valid only - with field mode. */ -@@ -132,6 +206,9 @@ - - /* The delimeter character for field mode. */ - static unsigned char delim; -+#if HAVE_WCHAR_H -+static wchar_t wcdelim; -+#endif - - /* True if the --output-delimiter=STRING option was specified. */ - static bool output_delimiter_specified; -@@ -205,7 +282,7 @@ - -f, --fields=LIST select only these fields; also print any line\n\ - that contains no delimiter character, unless\n\ - the -s option is specified\n\ -- -n (ignored)\n\ -+ -n with -b: don't split multibyte characters\n\ - "), stdout); - fputs (_("\ - --complement complement the set of selected bytes, characters\n\ -@@ -360,7 +437,7 @@ - in_digits = false; - /* Starting a range. */ - if (dash_found) -- FATAL_ERROR (_("invalid byte or field list")); -+ FATAL_ERROR (_("invalid byte, character or field list")); - dash_found = true; - fieldstr++; - -@@ -385,14 +462,16 @@ - if (value == 0) - { - /* `n-'. From `initial' to end of line. */ -- eol_range_start = initial; -+ if (eol_range_start == 0 || -+ (eol_range_start != 0 && eol_range_start > initial)) -+ eol_range_start = initial; - field_found = true; - } - else - { - /* `m-n' or `-n' (1-n). */ - if (value < initial) -- FATAL_ERROR (_("invalid byte or field list")); -+ FATAL_ERROR (_("invalid byte, character or field list")); - - /* Is there already a range going to end of line? */ - if (eol_range_start != 0) -@@ -465,6 +544,9 @@ - if (operating_mode == byte_mode) - error (0, 0, - _("byte offset %s is too large"), quote (bad_num)); -+ else if (operating_mode == character_mode) -+ error (0, 0, -+ _("character offset %s is too large"), quote (bad_num)); - else - error (0, 0, - _("field number %s is too large"), quote (bad_num)); -@@ -475,7 +557,7 @@ - fieldstr++; +@@ -182,6 +199,7 @@ + stops = num_start + len - 1; + } } ++ else -- FATAL_ERROR (_("invalid byte or field list")); -+ FATAL_ERROR (_("invalid byte, character or field list")); - } - - max_range_endpoint = 0; -@@ -568,6 +650,63 @@ + { + error (0, 0, _("tab size contains invalid character(s): %s"), +@@ -364,6 +382,142 @@ } } +#if HAVE_MBRTOWC -+/* This function is in use for the following case. -+ -+ 1. Read from the stream STREAM, printing to standard output any selected -+ characters. -+ -+ 2. Read from stream STREAM, printing to standard output any selected bytes, -+ without splitting multibyte characters. */ -+ +static void -+cut_characters_or_cut_bytes_no_split (FILE *stream) ++expand_multibyte (void) +{ -+ int idx; /* number of bytes or characters in the line so far. */ ++ FILE *fp; /* Input strem. */ ++ mbstate_t i_state; /* Current shift state of the input stream. */ ++ mbstate_t i_state_bak; /* Back up the I_STATE. */ ++ mbstate_t o_state; /* Current shift state of the output stream. */ + char buf[MB_LEN_MAX + BUFSIZ]; /* For spooling a read byte sequence. */ -+ char *bufpos; /* Next read position of BUF. */ -+ size_t buflen; /* The length of the byte sequence in buf. */ -+ wint_t wc; /* A gotten wide character. */ -+ size_t mblength; /* The byte size of a multibyte character which shows -+ as same character as WC. */ -+ mbstate_t state; /* State of the stream. */ -+ int convfail; /* 1, when conversion is failed. Otherwise 0. */ ++ char *bufpos; /* Next read position of BUF. */ ++ size_t buflen = 0; /* The length of the byte sequence in buf. */ ++ wchar_t wc; /* A gotten wide character. */ ++ size_t mblength; /* The byte size of a multibyte character ++ which shows as same character as WC. */ ++ int tab_index = 0; /* Index in `tab_list' of next tabstop. */ ++ int column = 0; /* Column on screen of the next char. */ ++ int next_tab_column; /* Column the next tab stop is on. */ ++ int convert = 1; /* If nonzero, perform translations. */ + -+ idx = 0; -+ buflen = 0; -+ bufpos = buf; -+ memset (&state, '\0', sizeof(mbstate_t)); ++ fp = next_file ((FILE *) NULL); ++ if (fp == NULL) ++ return; + -+ while (1) ++ memset (&o_state, '\0', sizeof(mbstate_t)); ++ memset (&i_state, '\0', sizeof(mbstate_t)); ++ ++ for (;;) + { -+ REFILL_BUFFER (buf, bufpos, buflen, stream); -+ -+ GET_NEXT_WC_FROM_BUFFER (wc, bufpos, buflen, mblength, state, convfail); -+ -+ if (wc == WEOF) ++ /* Refill the buffer BUF. */ ++ if (buflen < MB_LEN_MAX && !feof(fp) && !ferror(fp)) + { -+ if (idx > 0) -+ putchar ('\n'); ++ memmove (buf, bufpos, buflen); ++ buflen += fread (buf + buflen, sizeof(char), BUFSIZ, fp); ++ bufpos = buf; ++ } ++ ++ /* No character is left in BUF. */ ++ if (buflen < 1) ++ { ++ fp = next_file (fp); ++ ++ if (fp == NULL) ++ break; /* No more files. */ ++ else ++ { ++ memset (&i_state, '\0', sizeof(mbstate_t)); ++ continue; ++ } ++ } ++ ++ /* Get a wide character. */ ++ i_state_bak = i_state; ++ mblength = mbrtowc (&wc, bufpos, buflen, &i_state); ++ ++ switch (mblength) ++ { ++ case (size_t)-1: /* illegal byte sequence. */ ++ case (size_t)-2: ++ mblength = 1; ++ i_state = i_state_bak; ++ if (convert) ++ { ++ ++column; ++ if (convert_entire_line == 0) ++ convert = 0; ++ } ++ putchar (*bufpos); + break; -+ } -+ else if (wc == L'\n') -+ { -+ putchar ('\n'); -+ idx = 0; -+ } -+ else -+ { -+ idx += (operating_mode == byte_mode) ? mblength : 1; -+ if (print_kth (idx, NULL)) -+ fwrite (bufpos, mblength, sizeof(char), stdout); -+ } + ++ case 0: /* null. */ ++ mblength = 1; ++ if (convert && convert_entire_line == 0) ++ convert = 0; ++ putchar ('\0'); ++ break; ++ ++ default: ++ if (wc == L'\n') /* LF. */ ++ { ++ tab_index = 0; ++ column = 0; ++ convert = 1; ++ putchar ('\n'); ++ } ++ else if (wc == L'\t' && convert) /* Tab. */ ++ { ++ if (tab_size == 0) ++ { ++ /* Do not let tab_index == first_free_tab; ++ stop when it is 1 less. */ ++ while (tab_index < first_free_tab - 1 ++ && column >= tab_list[tab_index]) ++ tab_index++; ++ next_tab_column = tab_list[tab_index]; ++ if (tab_index < first_free_tab - 1) ++ tab_index++; ++ if (column >= next_tab_column) ++ next_tab_column = column + 1; ++ } ++ else ++ next_tab_column = column + tab_size - column % tab_size; ++ ++ while (column < next_tab_column) ++ { ++ putchar (' '); ++ ++column; ++ } ++ } ++ else /* Others. */ ++ { ++ if (convert) ++ { ++ if (wc == L'\b') ++ { ++ if (column > 0) ++ --column; ++ } ++ else ++ { ++ int width; /* The width of WC. */ ++ ++ width = wcwidth (wc); ++ column += (width > 0) ? width : 0; ++ if (convert_entire_line == 0) ++ convert = 0; ++ } ++ } ++ fwrite (bufpos, sizeof(char), mblength, stdout); ++ } ++ } + buflen -= mblength; + bufpos += mblength; + } +} +#endif -+ - /* Read from stream STREAM, printing to standard output any selected fields. */ - - static void -@@ -689,13 +828,192 @@ - } - } - -+#if HAVE_MBRTOWC -+static void -+cut_fields_mb (FILE *stream) -+{ -+ int c; -+ unsigned int field_idx; -+ int found_any_selected_field; -+ int buffer_first_field; -+ int empty_input; -+ char buf[MB_LEN_MAX + BUFSIZ]; /* For spooling a read byte sequence. */ -+ char *bufpos; /* Next read position of BUF. */ -+ size_t buflen; /* The length of the byte sequence in buf. */ -+ wint_t wc = 0; /* A gotten wide character. */ -+ size_t mblength; /* The byte size of a multibyte character which shows -+ as same character as WC. */ -+ mbstate_t state; /* State of the stream. */ -+ int convfail; /* 1, when conversion is failed. Otherwise 0. */ + -+ found_any_selected_field = 0; -+ field_idx = 1; -+ bufpos = buf; -+ buflen = 0; -+ memset (&state, '\0', sizeof(mbstate_t)); -+ -+ c = getc (stream); -+ empty_input = (c == EOF); -+ if (c != EOF) -+ ungetc (c, stream); -+ else -+ wc = WEOF; -+ -+ /* To support the semantics of the -s flag, we may have to buffer -+ all of the first field to determine whether it is `delimited.' -+ But that is unnecessary if all non-delimited lines must be printed -+ and the first field has been selected, or if non-delimited lines -+ must be suppressed and the first field has *not* been selected. -+ That is because a non-delimited line has exactly one field. */ -+ buffer_first_field = (suppress_non_delimited ^ !print_kth (1, NULL)); -+ -+ while (1) -+ { -+ if (field_idx == 1 && buffer_first_field) -+ { -+ int len = 0; -+ -+ while (1) -+ { -+ REFILL_BUFFER (buf, bufpos, buflen, stream); -+ -+ GET_NEXT_WC_FROM_BUFFER -+ (wc, bufpos, buflen, mblength, state, convfail); -+ -+ if (wc == WEOF) -+ break; -+ -+ field_1_buffer = xrealloc (field_1_buffer, len + mblength); -+ memcpy (field_1_buffer + len, bufpos, mblength); -+ len += mblength; -+ buflen -= mblength; -+ bufpos += mblength; -+ -+ if (!convfail && (wc == L'\n' || wc == wcdelim)) -+ break; -+ } -+ -+ if (wc == WEOF) -+ break; -+ -+ /* If the first field extends to the end of line (it is not -+ delimited) and we are printing all non-delimited lines, -+ print this one. */ -+ if (convfail || (!convfail && wc != wcdelim)) -+ { -+ if (suppress_non_delimited) -+ { -+ /* Empty. */ -+ } -+ else -+ { -+ fwrite (field_1_buffer, sizeof (char), len, stdout); -+ /* Make sure the output line is newline terminated. */ -+ if (convfail || (!convfail && wc != L'\n')) -+ putchar ('\n'); -+ } -+ continue; -+ } -+ -+ if (print_kth (1, NULL)) -+ { -+ /* Print the field, but not the trailing delimiter. */ -+ fwrite (field_1_buffer, sizeof (char), len - 1, stdout); -+ found_any_selected_field = 1; -+ } -+ ++field_idx; -+ } -+ -+ if (wc != WEOF) -+ { -+ if (print_kth (field_idx, NULL)) -+ { -+ if (found_any_selected_field) -+ { -+ fwrite (output_delimiter_string, sizeof (char), -+ output_delimiter_length, stdout); -+ } -+ found_any_selected_field = 1; -+ } -+ -+ while (1) -+ { -+ REFILL_BUFFER (buf, bufpos, buflen, stream); -+ -+ GET_NEXT_WC_FROM_BUFFER -+ (wc, bufpos, buflen, mblength, state, convfail); -+ -+ if (wc == WEOF) -+ break; -+ else if (!convfail && (wc == wcdelim || wc == L'\n')) -+ { -+ buflen -= mblength; -+ bufpos += mblength; -+ break; -+ } -+ -+ if (print_kth (field_idx, NULL)) -+ fwrite (bufpos, mblength, sizeof(char), stdout); -+ -+ buflen -= mblength; -+ bufpos += mblength; -+ } -+ } -+ -+ if ((!convfail || wc == L'\n') && buflen < 1) -+ wc = WEOF; -+ -+ if (!convfail && wc == wcdelim) -+ ++field_idx; -+ else if (wc == WEOF || (!convfail && wc == L'\n')) -+ { -+ if (found_any_selected_field -+ || (!empty_input && !(suppress_non_delimited && field_idx == 1))) -+ putchar ('\n'); -+ if (wc == WEOF) -+ break; -+ field_idx = 1; -+ found_any_selected_field = 0; -+ } -+ } -+} -+#endif -+ - static void - cut_stream (FILE *stream) + int + main (int argc, char **argv) { -- if (operating_mode == byte_mode) -- cut_bytes (stream); +@@ -428,7 +582,12 @@ + + file_list = (optind < argc ? &argv[optind] : stdin_argv); + +- expand (); +#if HAVE_MBRTOWC -+ if (MB_CUR_MAX > 1 && !force_singlebyte_mode) -+ { -+ switch (operating_mode) -+ { -+ case byte_mode: -+ if (byte_mode_character_aware) -+ cut_characters_or_cut_bytes_no_split (stream); -+ else -+ cut_bytes (stream); -+ break; -+ -+ case character_mode: -+ cut_characters_or_cut_bytes_no_split (stream); -+ break; -+ -+ case field_mode: -+ cut_fields_mb (stream); -+ break; -+ -+ default: -+ abort (); -+ } -+ } - else -- cut_fields (stream); ++ if (MB_CUR_MAX > 1) ++ expand_multibyte (); ++ else +#endif -+ { -+ if (operating_mode == field_mode) -+ cut_fields (stream); -+ else -+ cut_bytes (stream); -+ } - } ++ expand (); - /* Process file FILE to standard output. -@@ -745,6 +1063,8 @@ - bool ok; - bool delim_specified = false; - char *spec_list_string IF_LINT(= NULL); -+ char mbdelim[MB_LEN_MAX + 1]; -+ size_t delimlen = 0; - - initialize_main (&argc, &argv); - program_name = argv[0]; -@@ -767,7 +1087,6 @@ - switch (optc) - { - case 'b': -- case 'c': - /* Build the byte list. */ - if (operating_mode != undefined_mode) - FATAL_ERROR (_("only one type of list may be specified")); -@@ -775,6 +1094,14 @@ - spec_list_string = optarg; - break; - -+ case 'c': -+ /* Build the character list. */ -+ if (operating_mode != undefined_mode) -+ FATAL_ERROR (_("only one type of list may be specified")); -+ operating_mode = character_mode; -+ spec_list_string = optarg; -+ break; -+ - case 'f': - /* Build the field list. */ - if (operating_mode != undefined_mode) -@@ -786,10 +1113,35 @@ - case 'd': - /* New delimiter. */ - /* Interpret -d '' to mean `use the NUL byte as the delimiter.' */ -- if (optarg[0] != '\0' && optarg[1] != '\0') -- FATAL_ERROR (_("the delimiter must be a single character")); -- delim = optarg[0]; -- delim_specified = true; -+#if HAVE_MBRTOWC -+ { -+ if(MB_CUR_MAX > 1) -+ { -+ mbstate_t state; -+ -+ memset (&state, '\0', sizeof(mbstate_t)); -+ delimlen = mbrtowc (&wcdelim, optarg, strnlen(optarg, MB_LEN_MAX), &state); -+ -+ if (delimlen == (size_t)-1 || delimlen == (size_t)-2) -+ ++force_singlebyte_mode; -+ else -+ { -+ delimlen = (delimlen < 1) ? 1 : delimlen; -+ if (wcdelim != L'\0' && *(optarg + delimlen) != '\0') -+ FATAL_ERROR (_("the delimiter must be a single character")); -+ memcpy (mbdelim, optarg, delimlen); -+ } -+ } -+ -+ if (MB_CUR_MAX <= 1 || force_singlebyte_mode) -+#endif -+ { -+ if (optarg[0] != '\0' && optarg[1] != '\0') -+ FATAL_ERROR (_("the delimiter must be a single character")); -+ delim = (unsigned char) optarg[0]; -+ } -+ delim_specified = true; -+ } - break; - - case OUTPUT_DELIMITER_OPTION: -@@ -802,6 +1154,7 @@ - break; - - case 'n': -+ byte_mode_character_aware = 1; - break; - - case 's': -@@ -824,7 +1177,7 @@ - if (operating_mode == undefined_mode) - FATAL_ERROR (_("you must specify a list of bytes, characters, or fields")); - -- if (delim != '\0' && operating_mode != field_mode) -+ if (delim_specified && operating_mode != field_mode) - FATAL_ERROR (_("an input delimiter may be specified only\ - when operating on fields")); - -@@ -851,15 +1204,34 @@ - } - - if (!delim_specified) -- delim = '\t'; -+ { -+ delim = '\t'; -+#ifdef HAVE_MBRTOWC -+ wcdelim = L'\t'; -+ mbdelim[0] = '\t'; -+ mbdelim[1] = '\0'; -+ delimlen = 1; -+#endif -+ } - - if (output_delimiter_string == NULL) - { -- static char dummy[2]; -- dummy[0] = delim; -- dummy[1] = '\0'; -- output_delimiter_string = dummy; -- output_delimiter_length = 1; -+#ifdef HAVE_MBRTOWC -+ if (MB_CUR_MAX > 1 && !force_singlebyte_mode) -+ { -+ output_delimiter_string = xstrdup(mbdelim); -+ output_delimiter_length = delimlen; -+ } -+ -+ if (MB_CUR_MAX <= 1 || force_singlebyte_mode) -+#endif -+ { -+ static char dummy[2]; -+ dummy[0] = delim; -+ dummy[1] = '\0'; -+ output_delimiter_string = dummy; -+ output_delimiter_length = 1; -+ } - } - - if (optind == argc) ---- coreutils-5.93/src/pr.c 2005-12-23 08:53:01.000000000 +0000 -+++ coreutils-5.95/src/pr.c 2006-05-15 13:45:56.000000000 +0100 + if (have_read_stdin && fclose (stdin) != 0) + error (EXIT_FAILURE, errno, "-"); +--- coreutils-5.95/src/pr.c.i18n 2005-09-16 08:50:33.000000000 +0100 ++++ coreutils-5.95/src/pr.c 2006-05-15 15:09:08.000000000 +0100 @@ -313,6 +313,32 @@ #include @@ -846,7 +425,7 @@ use_col_separator = true; if (optarg) separator_string (optarg); -@@ -1188,17 +1279,52 @@ +@@ -1188,10 +1279,45 @@ a number. */ static void @@ -894,14 +473,6 @@ if (*arg) { long int tmp_long; - We've to use 8 as the lower limit, if we use chars_per_default_tab = 8 - to expand a tab which is not an input_tab-char. */ - free (clump_buff); -- clump_buff = xmalloc (MAX (8, chars_per_input_tab)); -+ clump_buff = xmalloc (MAX (80, chars_per_input_tab)); - } - - /* Open the necessary files, @@ -1256,7 +1382,7 @@ else col_sep_string = column_separator; @@ -934,6 +505,15 @@ if (chars_per_column < 1) error (EXIT_FAILURE, 0, _("page width too narrow")); +@@ -1324,7 +1450,7 @@ + We've to use 8 as the lower limit, if we use chars_per_default_tab = 8 + to expand a tab which is not an input_tab-char. */ + free (clump_buff); +- clump_buff = xmalloc (MAX (8, chars_per_input_tab)); ++ clump_buff = xmalloc (MAX (80, chars_per_input_tab)); + } + + /* Open the necessary files, @@ -1432,7 +1558,7 @@ /* Enlarge p->start_position of first column to use the same form of @@ -1347,982 +927,8 @@ /* We've just printed some files and need to clean up things before looking for more options and printing the next batch of files. ---- coreutils-5.93/src/uniq.c.i18n 2005-07-05 07:32:54.000000000 +0100 -+++ coreutils-5.93/src/uniq.c 2005-12-23 08:53:01.000000000 +0000 -@@ -23,6 +23,16 @@ - #include - #include - -+/* Get mbstate_t, mbrtowc(). */ -+#if HAVE_WCHAR_H -+# include -+#endif -+ -+/* Get isw* functions. */ -+#if HAVE_WCTYPE_H -+# include -+#endif -+ - #include "system.h" - #include "argmatch.h" - #include "linebuffer.h" -@@ -32,7 +42,19 @@ - #include "quote.h" - #include "xmemcoll.h" - #include "xstrtol.h" --#include "memcasecmp.h" -+#include "xmemcoll.h" -+ -+/* MB_LEN_MAX is incorrectly defined to be 1 in at least one GCC -+ installation; work around this configuration error. */ -+#if !defined MB_LEN_MAX || MB_LEN_MAX < 2 -+# define MB_LEN_MAX 16 -+#endif -+ -+/* Some systems, like BeOS, have multibyte encodings but lack mbstate_t. */ -+#if HAVE_MBRTOWC && defined mbstate_t -+# define mbrtowc(pwc, s, n, ps) (mbrtowc) (pwc, s, n, 0) -+#endif -+ - - /* The official name of this program (e.g., no `g' prefix). */ - #define PROGRAM_NAME "uniq" -@@ -109,6 +131,10 @@ - /* Select whether/how to delimit groups of duplicate lines. */ - static enum delimit_method delimit_groups; - -+/* Function pointers. */ -+static char * -+(*find_field) (struct linebuffer *line); -+ - static struct option const longopts[] = - { - {"count", no_argument, NULL, 'c'}, -@@ -189,7 +215,7 @@ - return a pointer to the beginning of the line's field to be compared. */ - - static char * --find_field (const struct linebuffer *line) -+find_field_uni (struct linebuffer *line) - { - size_t count; - char *lp = line->buffer; -@@ -210,6 +236,83 @@ - return lp + i; - } - -+#if HAVE_MBRTOWC -+ -+# define MBCHAR_TO_WCHAR(WC, MBLENGTH, LP, POS, SIZE, STATEP, CONVFAIL) \ -+ do \ -+ { \ -+ mbstate_t state_bak; \ -+ \ -+ CONVFAIL = 0; \ -+ state_bak = *STATEP; \ -+ \ -+ MBLENGTH = mbrtowc (&WC, LP + POS, SIZE - POS, STATEP); \ -+ \ -+ switch (MBLENGTH) \ -+ { \ -+ case (size_t)-2: \ -+ case (size_t)-1: \ -+ *STATEP = state_bak; \ -+ CONVFAIL++; \ -+ /* Fall through */ \ -+ case 0: \ -+ MBLENGTH = 1; \ -+ } \ -+ } \ -+ while (0) -+ -+static char * -+find_field_multi (struct linebuffer *line) -+{ -+ size_t count; -+ char *lp = line->buffer; -+ size_t size = line->length - 1; -+ size_t pos; -+ size_t mblength; -+ wchar_t wc; -+ mbstate_t *statep; -+ int convfail; -+ -+ pos = 0; -+ statep = &(line->state); -+ -+ /* skip fields. */ -+ for (count = 0; count < skip_fields && pos < size; count++) -+ { -+ while (pos < size) -+ { -+ MBCHAR_TO_WCHAR (wc, mblength, lp, pos, size, statep, convfail); -+ -+ if (convfail || !iswblank (wc)) -+ { -+ pos += mblength; -+ break; -+ } -+ pos += mblength; -+ } -+ -+ while (pos < size) -+ { -+ MBCHAR_TO_WCHAR (wc, mblength, lp, pos, size, statep, convfail); -+ -+ if (!convfail && iswblank (wc)) -+ break; -+ -+ pos += mblength; -+ } -+ } -+ -+ /* skip fields. */ -+ for (count = 0; count < skip_chars && pos < size; count++) -+ { -+ MBCHAR_TO_WCHAR (wc, mblength, lp, pos, size, statep, convfail); -+ pos += mblength; -+ } -+ -+ return lp + pos; -+} -+#endif -+ - /* Return false if two strings OLD and NEW match, true if not. - OLD and NEW point not to the beginnings of the lines - but rather to the beginnings of the fields to compare. -@@ -218,6 +321,8 @@ - static bool - different (char *old, char *new, size_t oldlen, size_t newlen) - { -+ char *copy_old, *copy_new; -+ - if (check_chars < oldlen) - oldlen = check_chars; - if (check_chars < newlen) -@@ -225,14 +330,92 @@ - - if (ignore_case) - { -- /* FIXME: This should invoke strcoll somehow. */ -- return oldlen != newlen || memcasecmp (old, new, oldlen); -+ size_t i; -+ -+ copy_old = alloca (oldlen + 1); -+ copy_new = alloca (oldlen + 1); -+ -+ for (i = 0; i < oldlen; i++) -+ { -+ copy_old[i] = toupper (old[i]); -+ copy_new[i] = toupper (new[i]); -+ } - } -- else if (hard_LC_COLLATE) -- return xmemcoll (old, oldlen, new, newlen) != 0; - else -- return oldlen != newlen || memcmp (old, new, oldlen); -+ { -+ copy_old = (char *)old; -+ copy_new = (char *)new; -+ } -+ -+ return xmemcoll (copy_old, oldlen, copy_new, newlen); -+} -+ -+#if HAVE_MBRTOWC -+static int -+different_multi (const char *old, const char *new, size_t oldlen, size_t newlen, mbstate_t oldstate, mbstate_t newstate) -+{ -+ size_t i, j, chars; -+ const char *str[2]; -+ char *copy[2]; -+ size_t len[2]; -+ mbstate_t state[2]; -+ size_t mblength; -+ wchar_t wc, uwc; -+ mbstate_t state_bak; -+ -+ str[0] = old; -+ str[1] = new; -+ len[0] = oldlen; -+ len[1] = newlen; -+ state[0] = oldstate; -+ state[1] = newstate; -+ -+ for (i = 0; i < 2; i++) -+ { -+ copy[i] = alloca (len[i] + 1); -+ -+ for (j = 0, chars = 0; j < len[i] && chars < check_chars; chars++) -+ { -+ state_bak = state[i]; -+ mblength = mbrtowc (&wc, str[i] + j, len[i] - j, &(state[i])); -+ -+ switch (mblength) -+ { -+ case (size_t)-1: -+ case (size_t)-2: -+ state[i] = state_bak; -+ /* Fall through */ -+ case 0: -+ mblength = 1; -+ break; -+ -+ default: -+ if (ignore_case) -+ { -+ uwc = towupper (wc); -+ -+ if (uwc != wc) -+ { -+ mbstate_t state_wc; -+ -+ memset (&state_wc, '\0', sizeof(mbstate_t)); -+ wcrtomb (copy[i] + j, uwc, &state_wc); -+ } -+ else -+ memcpy (copy[i] + j, str[i] + j, mblength); -+ } -+ else -+ memcpy (copy[i] + j, str[i] + j, mblength); -+ } -+ j += mblength; -+ } -+ copy[i][j] = '\0'; -+ len[i] = j; -+ } -+ -+ return xmemcoll (copy[0], len[0], copy[1], len[1]); - } -+#endif - - /* Output the line in linebuffer LINE to standard output - provided that the switches say it should be output. -@@ -286,15 +469,43 @@ - { - char *prevfield IF_LINT (= NULL); - size_t prevlen IF_LINT (= 0); -+#if HAVE_MBRTOWC -+ mbstate_t prevstate; -+ -+ memset (&prevstate, '\0', sizeof (mbstate_t)); -+#endif - - while (!feof (stdin)) - { - char *thisfield; - size_t thislen; -+#if HAVE_MBRTOWC -+ mbstate_t thisstate; -+#endif -+ - if (readlinebuffer (thisline, stdin) == 0) - break; - thisfield = find_field (thisline); - thislen = thisline->length - 1 - (thisfield - thisline->buffer); -+#if HAVE_MBRTOWC -+ if (MB_CUR_MAX > 1) -+ { -+ thisstate = thisline->state; -+ -+ if (prevline->length == 0 || different_multi -+ (thisfield, prevfield, thislen, prevlen, thisstate, prevstate)) -+ { -+ fwrite (thisline->buffer, sizeof (char), -+ thisline->length, stdout); -+ -+ SWAP_LINES (prevline, thisline); -+ prevfield = thisfield; -+ prevlen = thislen; -+ prevstate = thisstate; -+ } -+ } -+ else -+#endif - if (prevline->length == 0 - || different (thisfield, prevfield, thislen, prevlen)) - { -@@ -313,17 +524,26 @@ - size_t prevlen; - uintmax_t match_count = 0; - bool first_delimiter = true; -+#if HAVE_MBRTOWC -+ mbstate_t prevstate; -+#endif - - if (readlinebuffer (prevline, stdin) == 0) - goto closefiles; - prevfield = find_field (prevline); - prevlen = prevline->length - 1 - (prevfield - prevline->buffer); -+#if HAVE_MBRTOWC -+ prevstate = prevline->state; -+#endif - - while (!feof (stdin)) - { - bool match; - char *thisfield; - size_t thislen; -+#if HAVE_MBRTOWC -+ mbstate_t thisstate; -+#endif - if (readlinebuffer (thisline, stdin) == 0) - { - if (ferror (stdin)) -@@ -332,6 +552,15 @@ - } - thisfield = find_field (thisline); - thislen = thisline->length - 1 - (thisfield - thisline->buffer); -+#if HAVE_MBRTOWC -+ if (MB_CUR_MAX > 1) -+ { -+ thisstate = thisline->state; -+ match = !different_multi (thisfield, prevfield, -+ thislen, prevlen, thisstate, prevstate); -+ } -+ else -+#endif - match = !different (thisfield, prevfield, thislen, prevlen); - match_count += match; - -@@ -364,6 +593,9 @@ - SWAP_LINES (prevline, thisline); - prevfield = thisfield; - prevlen = thislen; -+#if HAVE_MBRTOWC -+ prevstate = thisstate; -+#endif - if (!match) - match_count = 0; - } -@@ -408,6 +640,19 @@ - - atexit (close_stdout); - -+#if HAVE_MBRTOWC -+ if (MB_CUR_MAX > 1) -+ { -+ find_field = find_field_multi; -+ } -+ else -+#endif -+ { -+ find_field = find_field_uni; -+ } -+ -+ -+ - skip_chars = 0; - skip_fields = 0; - check_chars = SIZE_MAX; ---- coreutils-5.93/src/expand.c.i18n 2005-08-12 08:16:25.000000000 +0100 -+++ coreutils-5.93/src/expand.c 2005-12-23 08:53:01.000000000 +0000 -@@ -38,11 +38,28 @@ - #include - #include - #include -+ -+/* Get mbstate_t, mbrtowc(), wcwidth(). */ -+#if HAVE_WCHAR_H -+# include -+#endif -+ - #include "system.h" - #include "error.h" - #include "quote.h" - #include "xstrndup.h" - -+/* MB_LEN_MAX is incorrectly defined to be 1 in at least one GCC -+ installation; work around this configuration error. */ -+#if !defined MB_LEN_MAX || MB_LEN_MAX < 2 -+# define MB_LEN_MAX 16 -+#endif -+ -+/* Some systems, like BeOS, have multibyte encodings but lack mbstate_t. */ -+#if HAVE_MBRTOWC && defined mbstate_t -+# define mbrtowc(pwc, s, n, ps) (mbrtowc) (pwc, s, n, 0) -+#endif -+ - /* The official name of this program (e.g., no `g' prefix). */ - #define PROGRAM_NAME "expand" - -@@ -182,6 +199,7 @@ - stops = num_start + len - 1; - } - } -+ - else - { - error (0, 0, _("tab size contains invalid character(s): %s"), -@@ -364,6 +382,142 @@ - } - } - -+#if HAVE_MBRTOWC -+static void -+expand_multibyte (void) -+{ -+ FILE *fp; /* Input strem. */ -+ mbstate_t i_state; /* Current shift state of the input stream. */ -+ mbstate_t i_state_bak; /* Back up the I_STATE. */ -+ mbstate_t o_state; /* Current shift state of the output stream. */ -+ char buf[MB_LEN_MAX + BUFSIZ]; /* For spooling a read byte sequence. */ -+ char *bufpos; /* Next read position of BUF. */ -+ size_t buflen = 0; /* The length of the byte sequence in buf. */ -+ wchar_t wc; /* A gotten wide character. */ -+ size_t mblength; /* The byte size of a multibyte character -+ which shows as same character as WC. */ -+ int tab_index = 0; /* Index in `tab_list' of next tabstop. */ -+ int column = 0; /* Column on screen of the next char. */ -+ int next_tab_column; /* Column the next tab stop is on. */ -+ int convert = 1; /* If nonzero, perform translations. */ -+ -+ fp = next_file ((FILE *) NULL); -+ if (fp == NULL) -+ return; -+ -+ memset (&o_state, '\0', sizeof(mbstate_t)); -+ memset (&i_state, '\0', sizeof(mbstate_t)); -+ -+ for (;;) -+ { -+ /* Refill the buffer BUF. */ -+ if (buflen < MB_LEN_MAX && !feof(fp) && !ferror(fp)) -+ { -+ memmove (buf, bufpos, buflen); -+ buflen += fread (buf + buflen, sizeof(char), BUFSIZ, fp); -+ bufpos = buf; -+ } -+ -+ /* No character is left in BUF. */ -+ if (buflen < 1) -+ { -+ fp = next_file (fp); -+ -+ if (fp == NULL) -+ break; /* No more files. */ -+ else -+ { -+ memset (&i_state, '\0', sizeof(mbstate_t)); -+ continue; -+ } -+ } -+ -+ /* Get a wide character. */ -+ i_state_bak = i_state; -+ mblength = mbrtowc (&wc, bufpos, buflen, &i_state); -+ -+ switch (mblength) -+ { -+ case (size_t)-1: /* illegal byte sequence. */ -+ case (size_t)-2: -+ mblength = 1; -+ i_state = i_state_bak; -+ if (convert) -+ { -+ ++column; -+ if (convert_entire_line == 0) -+ convert = 0; -+ } -+ putchar (*bufpos); -+ break; -+ -+ case 0: /* null. */ -+ mblength = 1; -+ if (convert && convert_entire_line == 0) -+ convert = 0; -+ putchar ('\0'); -+ break; -+ -+ default: -+ if (wc == L'\n') /* LF. */ -+ { -+ tab_index = 0; -+ column = 0; -+ convert = 1; -+ putchar ('\n'); -+ } -+ else if (wc == L'\t' && convert) /* Tab. */ -+ { -+ if (tab_size == 0) -+ { -+ /* Do not let tab_index == first_free_tab; -+ stop when it is 1 less. */ -+ while (tab_index < first_free_tab - 1 -+ && column >= tab_list[tab_index]) -+ tab_index++; -+ next_tab_column = tab_list[tab_index]; -+ if (tab_index < first_free_tab - 1) -+ tab_index++; -+ if (column >= next_tab_column) -+ next_tab_column = column + 1; -+ } -+ else -+ next_tab_column = column + tab_size - column % tab_size; -+ -+ while (column < next_tab_column) -+ { -+ putchar (' '); -+ ++column; -+ } -+ } -+ else /* Others. */ -+ { -+ if (convert) -+ { -+ if (wc == L'\b') -+ { -+ if (column > 0) -+ --column; -+ } -+ else -+ { -+ int width; /* The width of WC. */ -+ -+ width = wcwidth (wc); -+ column += (width > 0) ? width : 0; -+ if (convert_entire_line == 0) -+ convert = 0; -+ } -+ } -+ fwrite (bufpos, sizeof(char), mblength, stdout); -+ } -+ } -+ buflen -= mblength; -+ bufpos += mblength; -+ } -+} -+#endif -+ - int - main (int argc, char **argv) - { -@@ -428,7 +582,12 @@ - - file_list = (optind < argc ? &argv[optind] : stdin_argv); - -- expand (); -+#if HAVE_MBRTOWC -+ if (MB_CUR_MAX > 1) -+ expand_multibyte (); -+ else -+#endif -+ expand (); - - if (have_read_stdin && fclose (stdin) != 0) - error (EXIT_FAILURE, errno, "-"); ---- coreutils-5.93/src/fold.c.i18n 2005-08-12 08:29:38.000000000 +0100 -+++ coreutils-5.93/src/fold.c 2005-12-23 08:53:01.000000000 +0000 -@@ -23,11 +23,33 @@ - #include - #include - -+/* Get mbstate_t, mbrtowc(), wcwidth(). */ -+#if HAVE_WCHAR_H -+# include -+#endif -+ -+/* Get iswprint(), iswblank(), wcwidth(). */ -+#if HAVE_WCTYPE_H -+# include -+#endif -+ - #include "system.h" - #include "error.h" - #include "quote.h" - #include "xstrtol.h" - -+/* MB_LEN_MAX is incorrectly defined to be 1 in at least one GCC -+ installation; work around this configuration error. */ -+#if !defined MB_LEN_MAX || MB_LEN_MAX < 2 -+# undef MB_LEN_MAX -+# define MB_LEN_MAX 16 -+#endif -+ -+/* Some systems, like BeOS, have multibyte encodings but lack mbstate_t. */ -+#if HAVE_MBRTOWC && defined mbstate_t -+# define mbrtowc(pwc, s, n, ps) (mbrtowc) (pwc, s, n, 0) -+#endif -+ - #define TAB_WIDTH 8 - - /* The official name of this program (e.g., no `g' prefix). */ -@@ -35,23 +57,44 @@ - - #define AUTHORS "David MacKenzie" - -+#define FATAL_ERROR(Message) \ -+ do \ -+ { \ -+ error (0, 0, (Message)); \ -+ usage (2); \ -+ } \ -+ while (0) -+ -+enum operating_mode -+{ -+ /* Fold texts by columns that are at the given positions. */ -+ column_mode, -+ -+ /* Fold texts by bytes that are at the given positions. */ -+ byte_mode, -+ -+ /* Fold texts by characters that are at the given positions. */ -+ character_mode, -+}; -+ - /* The name this program was run with. */ - char *program_name; - -+/* The argument shows current mode. (Default: column_mode) */ -+static enum operating_mode operating_mode; -+ - /* If nonzero, try to break on whitespace. */ - static bool break_spaces; - --/* If nonzero, count bytes, not column positions. */ --static bool count_bytes; -- - /* If nonzero, at least one of the files we read was standard input. */ - static bool have_read_stdin; - --static char const shortopts[] = "bsw:0::1::2::3::4::5::6::7::8::9::"; -+static char const shortopts[] = "bcsw:0::1::2::3::4::5::6::7::8::9::"; - - static struct option const longopts[] = - { - {"bytes", no_argument, NULL, 'b'}, -+ {"characters", no_argument, NULL, 'c'}, - {"spaces", no_argument, NULL, 's'}, - {"width", required_argument, NULL, 'w'}, - {GETOPT_HELP_OPTION_DECL}, -@@ -81,6 +124,7 @@ - "), stdout); - fputs (_("\ - -b, --bytes count bytes rather than columns\n\ -+ -c, --characters count characters rather than columns\n\ - -s, --spaces break at spaces\n\ - -w, --width=WIDTH use WIDTH columns instead of 80\n\ - "), stdout); -@@ -98,7 +142,7 @@ - static size_t - adjust_column (size_t column, char c) - { -- if (!count_bytes) -+ if (operating_mode != byte_mode) - { - if (c == '\b') - { -@@ -117,35 +161,14 @@ - return column; - } - --/* Fold file FILENAME, or standard input if FILENAME is "-", -- to stdout, with maximum line length WIDTH. -- Return true if successful. */ -- --static bool --fold_file (char *filename, size_t width) -+static void -+fold_text (FILE *istream, size_t width, int *saved_errno) - { -- FILE *istream; - int c; - size_t column = 0; /* Screen column where next char will go. */ - size_t offset_out = 0; /* Index in `line_out' for next char. */ - static char *line_out = NULL; - static size_t allocated_out = 0; -- int saved_errno; -- -- if (STREQ (filename, "-")) -- { -- istream = stdin; -- have_read_stdin = true; -- } -- else -- istream = fopen (filename, "r"); -- -- if (istream == NULL) -- { -- error (0, errno, "%s", filename); -- return false; -- } -- - while ((c = getc (istream)) != EOF) - { - if (offset_out + 1 >= allocated_out) -@@ -172,6 +195,15 @@ - bool found_blank = false; - size_t logical_end = offset_out; - -+ /* If LINE_OUT has no wide character, -+ put a new wide character in LINE_OUT -+ if column is bigger than width. */ -+ if (offset_out == 0) -+ { -+ line_out[offset_out++] = c; -+ continue; -+ } -+ - /* Look for the last blank. */ - while (logical_end) - { -@@ -218,11 +250,225 @@ - line_out[offset_out++] = c; - } - -- saved_errno = errno; -+ *saved_errno = errno; -+ -+ if (offset_out) -+ fwrite (line_out, sizeof (char), (size_t) offset_out, stdout); -+ -+ free(line_out); -+} -+ -+#if HAVE_MBRTOWC -+static void -+fold_multibyte_text (FILE *istream, int width, int *saved_errno) -+{ -+ char buf[MB_LEN_MAX + BUFSIZ]; /* For spooling a read byte sequence. */ -+ size_t buflen = 0; /* The length of the byte sequence in buf. */ -+ char *bufpos; /* Next read position of BUF. */ -+ wint_t wc; /* A gotten wide character. */ -+ size_t mblength; /* The byte size of a multibyte character which shows -+ as same character as WC. */ -+ mbstate_t state, state_bak; /* State of the stream. */ -+ int convfail; /* 1, when conversion is failed. Otherwise 0. */ -+ -+ char *line_out = NULL; -+ size_t offset_out = 0; /* Index in `line_out' for next char. */ -+ size_t allocated_out = 0; -+ -+ int increment; -+ size_t column = 0; -+ -+ size_t last_blank_pos; -+ size_t last_blank_column; -+ int is_blank_seen; -+ int last_blank_increment; -+ int is_bs_following_last_blank; -+ size_t bs_following_last_blank_num; -+ int is_cr_after_last_blank; -+ -+#define CLEAR_FLAGS \ -+ do \ -+ { \ -+ last_blank_pos = 0; \ -+ last_blank_column = 0; \ -+ is_blank_seen = 0; \ -+ is_bs_following_last_blank = 0; \ -+ bs_following_last_blank_num = 0; \ -+ is_cr_after_last_blank = 0; \ -+ } \ -+ while (0) -+ -+#define START_NEW_LINE \ -+ do \ -+ { \ -+ putchar ('\n'); \ -+ column = 0; \ -+ offset_out = 0; \ -+ CLEAR_FLAGS; \ -+ } \ -+ while (0) -+ -+ CLEAR_FLAGS; -+ memset (&state, '\0', sizeof(mbstate_t)); -+ -+ for (;; bufpos += mblength, buflen -= mblength) -+ { -+ if (buflen < MB_LEN_MAX && !feof (istream) && !ferror (istream)) -+ { -+ memmove (buf, bufpos, buflen); -+ buflen += fread (buf + buflen, sizeof(char), BUFSIZ, istream); -+ bufpos = buf; -+ } -+ -+ if (buflen < 1) -+ break; -+ -+ /* Get a wide character. */ -+ convfail = 0; -+ state_bak = state; -+ mblength = mbrtowc ((wchar_t *)&wc, bufpos, buflen, &state); -+ -+ switch (mblength) -+ { -+ case (size_t)-1: -+ case (size_t)-2: -+ convfail++; -+ state = state_bak; -+ /* Fall through. */ -+ -+ case 0: -+ mblength = 1; -+ break; -+ } -+ -+rescan: -+ if (operating_mode == byte_mode) /* byte mode */ -+ increment = mblength; -+ else if (operating_mode == character_mode) /* character mode */ -+ increment = 1; -+ else /* column mode */ -+ { -+ if (convfail) -+ increment = 1; -+ else -+ { -+ switch (wc) -+ { -+ case L'\n': -+ fwrite (line_out, sizeof(char), offset_out, stdout); -+ START_NEW_LINE; -+ continue; -+ -+ case L'\b': -+ increment = (column > 0) ? -1 : 0; -+ break; -+ -+ case L'\r': -+ increment = -1 * column; -+ break; -+ -+ case L'\t': -+ increment = 8 - column % 8; -+ break; -+ -+ default: -+ increment = wcwidth (wc); -+ increment = (increment < 0) ? 0 : increment; -+ } -+ } -+ } -+ -+ if (column + increment > width && break_spaces && last_blank_pos) -+ { -+ fwrite (line_out, sizeof(char), last_blank_pos, stdout); -+ putchar ('\n'); -+ -+ offset_out = offset_out - last_blank_pos; -+ column = column - last_blank_column + ((is_cr_after_last_blank) -+ ? last_blank_increment : bs_following_last_blank_num); -+ memmove (line_out, line_out + last_blank_pos, offset_out); -+ CLEAR_FLAGS; -+ goto rescan; -+ } -+ -+ if (column + increment > width && column != 0) -+ { -+ fwrite (line_out, sizeof(char), offset_out, stdout); -+ START_NEW_LINE; -+ goto rescan; -+ } -+ -+ if (allocated_out < offset_out + mblength) -+ { -+ allocated_out += 1024; -+ line_out = xrealloc (line_out, allocated_out); -+ } -+ -+ memcpy (line_out + offset_out, bufpos, mblength); -+ offset_out += mblength; -+ column += increment; -+ -+ if (is_blank_seen && !convfail && wc == L'\r') -+ is_cr_after_last_blank = 1; -+ -+ if (is_bs_following_last_blank && !convfail && wc == L'\b') -+ ++bs_following_last_blank_num; -+ else -+ is_bs_following_last_blank = 0; -+ -+ if (break_spaces && !convfail && iswblank (wc)) -+ { -+ last_blank_pos = offset_out; -+ last_blank_column = column; -+ is_blank_seen = 1; -+ last_blank_increment = increment; -+ is_bs_following_last_blank = 1; -+ bs_following_last_blank_num = 0; -+ is_cr_after_last_blank = 0; -+ } -+ } -+ -+ *saved_errno = errno; - - if (offset_out) - fwrite (line_out, sizeof (char), (size_t) offset_out, stdout); - -+ free(line_out); -+} -+#endif -+ -+/* Fold file FILENAME, or standard input if FILENAME is "-", -+ to stdout, with maximum line length WIDTH. -+ Return 0 if successful, 1 if an error occurs. */ -+ -+static int -+fold_file (char *filename, int width) -+{ -+ FILE *istream; -+ int saved_errno; -+ -+ if (STREQ (filename, "-")) -+ { -+ istream = stdin; -+ have_read_stdin = 1; -+ } -+ else -+ istream = fopen (filename, "r"); -+ -+ if (istream == NULL) -+ { -+ error (0, errno, "%s", filename); -+ return 1; -+ } -+ -+ /* Define how ISTREAM is being folded. */ -+#if HAVE_MBRTOWC -+ if (MB_CUR_MAX > 1) -+ fold_multibyte_text (istream, width, &saved_errno); -+ else -+#endif -+ fold_text (istream, width, &saved_errno); -+ - if (ferror (istream)) - { - error (0, saved_errno, "%s", filename); -@@ -255,7 +501,8 @@ - - atexit (close_stdout); - -- break_spaces = count_bytes = have_read_stdin = false; -+ operating_mode = column_mode; -+ break_spaces = have_read_stdin = false; - - while ((optc = getopt_long (argc, argv, shortopts, longopts, NULL)) != -1) - { -@@ -264,7 +511,15 @@ - switch (optc) - { - case 'b': /* Count bytes rather than columns. */ -- count_bytes = true; -+ if (operating_mode != column_mode) -+ FATAL_ERROR (_("only one way of folding may be specified")); -+ operating_mode = byte_mode; -+ break; -+ -+ case 'c': -+ if (operating_mode != column_mode) -+ FATAL_ERROR (_("only one way of folding may be specified")); -+ operating_mode = character_mode; - break; - - case 's': /* Break at word boundaries. */ ---- coreutils-5.93/src/join.c.i18n 2005-08-12 08:16:25.000000000 +0100 -+++ coreutils-5.93/src/join.c 2005-12-23 08:53:01.000000000 +0000 +--- coreutils-5.95/src/join.c.i18n 2005-08-12 08:16:25.000000000 +0100 ++++ coreutils-5.95/src/join.c 2006-05-15 15:08:57.000000000 +0100 @@ -23,16 +23,30 @@ #include #include @@ -2818,262 +1424,8 @@ } break; ---- coreutils-5.93/src/unexpand.c.i18n 2005-08-12 08:16:25.000000000 +0100 -+++ coreutils-5.93/src/unexpand.c 2005-12-23 08:53:01.000000000 +0000 -@@ -39,11 +39,28 @@ - #include - #include - #include -+ -+/* Get mbstate_t, mbrtowc(), wcwidth(). */ -+#if HAVE_WCHAR_H -+# include -+#endif -+ - #include "system.h" - #include "error.h" - #include "quote.h" - #include "xstrndup.h" - -+/* MB_LEN_MAX is incorrectly defined to be 1 in at least one GCC -+ installation; work around this configuration error. */ -+#if !defined MB_LEN_MAX || MB_LEN_MAX < 2 -+# define MB_LEN_MAX 16 -+#endif -+ -+/* Some systems, like BeOS, have multibyte encodings but lack mbstate_t. */ -+#if HAVE_MBRTOWC && defined mbstate_t -+# define mbrtowc(pwc, s, n, ps) (mbrtowc) (pwc, s, n, 0) -+#endif -+ - /* The official name of this program (e.g., no `g' prefix). */ - #define PROGRAM_NAME "unexpand" - -@@ -110,6 +127,208 @@ - {NULL, 0, NULL, 0} - }; - -+static FILE *next_file (FILE *fp); -+ -+#if HAVE_MBRTOWC -+static void -+unexpand_multibyte (void) -+{ -+ FILE *fp; /* Input stream. */ -+ mbstate_t i_state; /* Current shift state of the input stream. */ -+ mbstate_t i_state_bak; /* Back up the I_STATE. */ -+ mbstate_t o_state; /* Current shift state of the output stream. */ -+ char buf[MB_LEN_MAX + BUFSIZ]; /* For spooling a read byte sequence. */ -+ char *bufpos; /* Next read position of BUF. */ -+ size_t buflen = 0; /* The length of the byte sequence in buf. */ -+ wint_t wc; /* A gotten wide character. */ -+ size_t mblength; /* The byte size of a multibyte character -+ which shows as same character as WC. */ -+ -+ /* Index in `tab_list' of next tabstop: */ -+ int tab_index = 0; /* For calculating width of pending tabs. */ -+ int print_tab_index = 0; /* For printing as many tabs as possible. */ -+ unsigned int column = 0; /* Column on screen of next char. */ -+ int next_tab_column; /* Column the next tab stop is on. */ -+ int convert = 1; /* If nonzero, perform translations. */ -+ unsigned int pending = 0; /* Pending columns of blanks. */ -+ -+ fp = next_file ((FILE *) NULL); -+ if (fp == NULL) -+ return; -+ -+ memset (&o_state, '\0', sizeof(mbstate_t)); -+ memset (&i_state, '\0', sizeof(mbstate_t)); -+ -+ for (;;) -+ { -+ if (buflen < MB_LEN_MAX && !feof(fp) && !ferror(fp)) -+ { -+ memmove (buf, bufpos, buflen); -+ buflen += fread (buf + buflen, sizeof(char), BUFSIZ, fp); -+ bufpos = buf; -+ } -+ -+ /* Get a wide character. */ -+ if (buflen < 1) -+ { -+ mblength = 1; -+ wc = WEOF; -+ } -+ else -+ { -+ i_state_bak = i_state; -+ mblength = mbrtowc ((wchar_t *)&wc, bufpos, buflen, &i_state); -+ } -+ -+ if (mblength == (size_t)-1 || mblength == (size_t)-2) -+ { -+ i_state = i_state_bak; -+ wc = L'\0'; -+ } -+ -+ if (wc == L' ' && convert && column < INT_MAX) -+ { -+ ++pending; -+ ++column; -+ } -+ else if (wc == L'\t' && convert) -+ { -+ if (tab_size == 0) -+ { -+ /* Do not let tab_index == first_free_tab; -+ stop when it is 1 less. */ -+ while (tab_index < first_free_tab - 1 -+ && column >= tab_list[tab_index]) -+ tab_index++; -+ next_tab_column = tab_list[tab_index]; -+ if (tab_index < first_free_tab - 1) -+ tab_index++; -+ if (column >= next_tab_column) -+ { -+ convert = 0; /* Ran out of tab stops. */ -+ goto flush_pend_mb; -+ } -+ } -+ else -+ { -+ next_tab_column = column + tab_size - column % tab_size; -+ } -+ pending += next_tab_column - column; -+ column = next_tab_column; -+ } -+ else -+ { -+flush_pend_mb: -+ /* Flush pending spaces. Print as many tabs as possible, -+ then print the rest as spaces. */ -+ if (pending == 1) -+ { -+ putchar (' '); -+ pending = 0; -+ } -+ column -= pending; -+ while (pending > 0) -+ { -+ if (tab_size == 0) -+ { -+ /* Do not let print_tab_index == first_free_tab; -+ stop when it is 1 less. */ -+ while (print_tab_index < first_free_tab - 1 -+ && column >= tab_list[print_tab_index]) -+ print_tab_index++; -+ next_tab_column = tab_list[print_tab_index]; -+ if (print_tab_index < first_free_tab - 1) -+ print_tab_index++; -+ } -+ else -+ { -+ next_tab_column = -+ column + tab_size - column % tab_size; -+ } -+ if (next_tab_column - column <= pending) -+ { -+ putchar ('\t'); -+ pending -= next_tab_column - column; -+ column = next_tab_column; -+ } -+ else -+ { -+ --print_tab_index; -+ column += pending; -+ while (pending != 0) -+ { -+ putchar (' '); -+ pending--; -+ } -+ } -+ } -+ -+ if (wc == WEOF) -+ { -+ fp = next_file (fp); -+ if (fp == NULL) -+ break; /* No more files. */ -+ else -+ { -+ memset (&i_state, '\0', sizeof(mbstate_t)); -+ continue; -+ } -+ } -+ -+ if (mblength == (size_t)-1 || mblength == (size_t)-2) -+ { -+ if (convert) -+ { -+ ++column; -+ if (convert_entire_line == 0) -+ convert = 0; -+ } -+ mblength = 1; -+ putchar (buf[0]); -+ } -+ else if (mblength == 0) -+ { -+ if (convert && convert_entire_line == 0) -+ convert = 0; -+ mblength = 1; -+ putchar ('\0'); -+ } -+ else -+ { -+ if (convert) -+ { -+ if (wc == L'\b') -+ { -+ if (column > 0) -+ --column; -+ } -+ else -+ { -+ int width; /* The width of WC. */ -+ -+ width = wcwidth (wc); -+ column += (width > 0) ? width : 0; -+ if (convert_entire_line == 0) -+ convert = 0; -+ } -+ } -+ -+ if (wc == L'\n') -+ { -+ tab_index = print_tab_index = 0; -+ column = pending = 0; -+ convert = 1; -+ } -+ fwrite (bufpos, sizeof(char), mblength, stdout); -+ } -+ } -+ buflen -= mblength; -+ bufpos += mblength; -+ } -+} -+#endif -+ -+ - void - usage (int status) - { -@@ -532,7 +751,12 @@ - - file_list = (optind < argc ? &argv[optind] : stdin_argv); - -- unexpand (); -+#if HAVE_MBRTOWC -+ if (MB_CUR_MAX > 1) -+ unexpand_multibyte (); -+ else -+#endif -+ unexpand (); - - if (have_read_stdin && fclose (stdin) != 0) - error (EXIT_FAILURE, errno, "-"); ---- coreutils-5.93/src/sort.c.i18n 2005-10-07 19:48:28.000000000 +0100 -+++ coreutils-5.93/src/sort.c 2005-12-23 10:38:44.000000000 +0000 +--- coreutils-5.95/src/sort.c.i18n 2005-10-07 19:48:28.000000000 +0100 ++++ coreutils-5.95/src/sort.c 2006-05-15 15:08:57.000000000 +0100 @@ -23,9 +23,18 @@ #include @@ -3910,8 +2262,1676 @@ } break; ---- /dev/null 2005-10-10 09:36:06.437701000 +0100 -+++ coreutils-5.93/tests/sort/sort-mb-tests 2005-12-23 08:53:01.000000000 +0000 +--- coreutils-5.95/src/uniq.c.i18n 2005-07-05 07:32:54.000000000 +0100 ++++ coreutils-5.95/src/uniq.c 2006-05-15 15:08:57.000000000 +0100 +@@ -23,6 +23,16 @@ + #include + #include + ++/* Get mbstate_t, mbrtowc(). */ ++#if HAVE_WCHAR_H ++# include ++#endif ++ ++/* Get isw* functions. */ ++#if HAVE_WCTYPE_H ++# include ++#endif ++ + #include "system.h" + #include "argmatch.h" + #include "linebuffer.h" +@@ -32,7 +42,19 @@ + #include "quote.h" + #include "xmemcoll.h" + #include "xstrtol.h" +-#include "memcasecmp.h" ++#include "xmemcoll.h" ++ ++/* MB_LEN_MAX is incorrectly defined to be 1 in at least one GCC ++ installation; work around this configuration error. */ ++#if !defined MB_LEN_MAX || MB_LEN_MAX < 2 ++# define MB_LEN_MAX 16 ++#endif ++ ++/* Some systems, like BeOS, have multibyte encodings but lack mbstate_t. */ ++#if HAVE_MBRTOWC && defined mbstate_t ++# define mbrtowc(pwc, s, n, ps) (mbrtowc) (pwc, s, n, 0) ++#endif ++ + + /* The official name of this program (e.g., no `g' prefix). */ + #define PROGRAM_NAME "uniq" +@@ -109,6 +131,10 @@ + /* Select whether/how to delimit groups of duplicate lines. */ + static enum delimit_method delimit_groups; + ++/* Function pointers. */ ++static char * ++(*find_field) (struct linebuffer *line); ++ + static struct option const longopts[] = + { + {"count", no_argument, NULL, 'c'}, +@@ -189,7 +215,7 @@ + return a pointer to the beginning of the line's field to be compared. */ + + static char * +-find_field (const struct linebuffer *line) ++find_field_uni (struct linebuffer *line) + { + size_t count; + char *lp = line->buffer; +@@ -210,6 +236,83 @@ + return lp + i; + } + ++#if HAVE_MBRTOWC ++ ++# define MBCHAR_TO_WCHAR(WC, MBLENGTH, LP, POS, SIZE, STATEP, CONVFAIL) \ ++ do \ ++ { \ ++ mbstate_t state_bak; \ ++ \ ++ CONVFAIL = 0; \ ++ state_bak = *STATEP; \ ++ \ ++ MBLENGTH = mbrtowc (&WC, LP + POS, SIZE - POS, STATEP); \ ++ \ ++ switch (MBLENGTH) \ ++ { \ ++ case (size_t)-2: \ ++ case (size_t)-1: \ ++ *STATEP = state_bak; \ ++ CONVFAIL++; \ ++ /* Fall through */ \ ++ case 0: \ ++ MBLENGTH = 1; \ ++ } \ ++ } \ ++ while (0) ++ ++static char * ++find_field_multi (struct linebuffer *line) ++{ ++ size_t count; ++ char *lp = line->buffer; ++ size_t size = line->length - 1; ++ size_t pos; ++ size_t mblength; ++ wchar_t wc; ++ mbstate_t *statep; ++ int convfail; ++ ++ pos = 0; ++ statep = &(line->state); ++ ++ /* skip fields. */ ++ for (count = 0; count < skip_fields && pos < size; count++) ++ { ++ while (pos < size) ++ { ++ MBCHAR_TO_WCHAR (wc, mblength, lp, pos, size, statep, convfail); ++ ++ if (convfail || !iswblank (wc)) ++ { ++ pos += mblength; ++ break; ++ } ++ pos += mblength; ++ } ++ ++ while (pos < size) ++ { ++ MBCHAR_TO_WCHAR (wc, mblength, lp, pos, size, statep, convfail); ++ ++ if (!convfail && iswblank (wc)) ++ break; ++ ++ pos += mblength; ++ } ++ } ++ ++ /* skip fields. */ ++ for (count = 0; count < skip_chars && pos < size; count++) ++ { ++ MBCHAR_TO_WCHAR (wc, mblength, lp, pos, size, statep, convfail); ++ pos += mblength; ++ } ++ ++ return lp + pos; ++} ++#endif ++ + /* Return false if two strings OLD and NEW match, true if not. + OLD and NEW point not to the beginnings of the lines + but rather to the beginnings of the fields to compare. +@@ -218,6 +321,8 @@ + static bool + different (char *old, char *new, size_t oldlen, size_t newlen) + { ++ char *copy_old, *copy_new; ++ + if (check_chars < oldlen) + oldlen = check_chars; + if (check_chars < newlen) +@@ -225,14 +330,92 @@ + + if (ignore_case) + { +- /* FIXME: This should invoke strcoll somehow. */ +- return oldlen != newlen || memcasecmp (old, new, oldlen); ++ size_t i; ++ ++ copy_old = alloca (oldlen + 1); ++ copy_new = alloca (oldlen + 1); ++ ++ for (i = 0; i < oldlen; i++) ++ { ++ copy_old[i] = toupper (old[i]); ++ copy_new[i] = toupper (new[i]); ++ } + } +- else if (hard_LC_COLLATE) +- return xmemcoll (old, oldlen, new, newlen) != 0; + else +- return oldlen != newlen || memcmp (old, new, oldlen); ++ { ++ copy_old = (char *)old; ++ copy_new = (char *)new; ++ } ++ ++ return xmemcoll (copy_old, oldlen, copy_new, newlen); ++} ++ ++#if HAVE_MBRTOWC ++static int ++different_multi (const char *old, const char *new, size_t oldlen, size_t newlen, mbstate_t oldstate, mbstate_t newstate) ++{ ++ size_t i, j, chars; ++ const char *str[2]; ++ char *copy[2]; ++ size_t len[2]; ++ mbstate_t state[2]; ++ size_t mblength; ++ wchar_t wc, uwc; ++ mbstate_t state_bak; ++ ++ str[0] = old; ++ str[1] = new; ++ len[0] = oldlen; ++ len[1] = newlen; ++ state[0] = oldstate; ++ state[1] = newstate; ++ ++ for (i = 0; i < 2; i++) ++ { ++ copy[i] = alloca (len[i] + 1); ++ ++ for (j = 0, chars = 0; j < len[i] && chars < check_chars; chars++) ++ { ++ state_bak = state[i]; ++ mblength = mbrtowc (&wc, str[i] + j, len[i] - j, &(state[i])); ++ ++ switch (mblength) ++ { ++ case (size_t)-1: ++ case (size_t)-2: ++ state[i] = state_bak; ++ /* Fall through */ ++ case 0: ++ mblength = 1; ++ break; ++ ++ default: ++ if (ignore_case) ++ { ++ uwc = towupper (wc); ++ ++ if (uwc != wc) ++ { ++ mbstate_t state_wc; ++ ++ memset (&state_wc, '\0', sizeof(mbstate_t)); ++ wcrtomb (copy[i] + j, uwc, &state_wc); ++ } ++ else ++ memcpy (copy[i] + j, str[i] + j, mblength); ++ } ++ else ++ memcpy (copy[i] + j, str[i] + j, mblength); ++ } ++ j += mblength; ++ } ++ copy[i][j] = '\0'; ++ len[i] = j; ++ } ++ ++ return xmemcoll (copy[0], len[0], copy[1], len[1]); + } ++#endif + + /* Output the line in linebuffer LINE to standard output + provided that the switches say it should be output. +@@ -286,15 +469,43 @@ + { + char *prevfield IF_LINT (= NULL); + size_t prevlen IF_LINT (= 0); ++#if HAVE_MBRTOWC ++ mbstate_t prevstate; ++ ++ memset (&prevstate, '\0', sizeof (mbstate_t)); ++#endif + + while (!feof (stdin)) + { + char *thisfield; + size_t thislen; ++#if HAVE_MBRTOWC ++ mbstate_t thisstate; ++#endif ++ + if (readlinebuffer (thisline, stdin) == 0) + break; + thisfield = find_field (thisline); + thislen = thisline->length - 1 - (thisfield - thisline->buffer); ++#if HAVE_MBRTOWC ++ if (MB_CUR_MAX > 1) ++ { ++ thisstate = thisline->state; ++ ++ if (prevline->length == 0 || different_multi ++ (thisfield, prevfield, thislen, prevlen, thisstate, prevstate)) ++ { ++ fwrite (thisline->buffer, sizeof (char), ++ thisline->length, stdout); ++ ++ SWAP_LINES (prevline, thisline); ++ prevfield = thisfield; ++ prevlen = thislen; ++ prevstate = thisstate; ++ } ++ } ++ else ++#endif + if (prevline->length == 0 + || different (thisfield, prevfield, thislen, prevlen)) + { +@@ -313,17 +524,26 @@ + size_t prevlen; + uintmax_t match_count = 0; + bool first_delimiter = true; ++#if HAVE_MBRTOWC ++ mbstate_t prevstate; ++#endif + + if (readlinebuffer (prevline, stdin) == 0) + goto closefiles; + prevfield = find_field (prevline); + prevlen = prevline->length - 1 - (prevfield - prevline->buffer); ++#if HAVE_MBRTOWC ++ prevstate = prevline->state; ++#endif + + while (!feof (stdin)) + { + bool match; + char *thisfield; + size_t thislen; ++#if HAVE_MBRTOWC ++ mbstate_t thisstate; ++#endif + if (readlinebuffer (thisline, stdin) == 0) + { + if (ferror (stdin)) +@@ -332,6 +552,15 @@ + } + thisfield = find_field (thisline); + thislen = thisline->length - 1 - (thisfield - thisline->buffer); ++#if HAVE_MBRTOWC ++ if (MB_CUR_MAX > 1) ++ { ++ thisstate = thisline->state; ++ match = !different_multi (thisfield, prevfield, ++ thislen, prevlen, thisstate, prevstate); ++ } ++ else ++#endif + match = !different (thisfield, prevfield, thislen, prevlen); + match_count += match; + +@@ -364,6 +593,9 @@ + SWAP_LINES (prevline, thisline); + prevfield = thisfield; + prevlen = thislen; ++#if HAVE_MBRTOWC ++ prevstate = thisstate; ++#endif + if (!match) + match_count = 0; + } +@@ -408,6 +640,19 @@ + + atexit (close_stdout); + ++#if HAVE_MBRTOWC ++ if (MB_CUR_MAX > 1) ++ { ++ find_field = find_field_multi; ++ } ++ else ++#endif ++ { ++ find_field = find_field_uni; ++ } ++ ++ ++ + skip_chars = 0; + skip_fields = 0; + check_chars = SIZE_MAX; +--- coreutils-5.95/src/unexpand.c.i18n 2005-08-12 08:16:25.000000000 +0100 ++++ coreutils-5.95/src/unexpand.c 2006-05-15 15:08:57.000000000 +0100 +@@ -39,11 +39,28 @@ + #include + #include + #include ++ ++/* Get mbstate_t, mbrtowc(), wcwidth(). */ ++#if HAVE_WCHAR_H ++# include ++#endif ++ + #include "system.h" + #include "error.h" + #include "quote.h" + #include "xstrndup.h" + ++/* MB_LEN_MAX is incorrectly defined to be 1 in at least one GCC ++ installation; work around this configuration error. */ ++#if !defined MB_LEN_MAX || MB_LEN_MAX < 2 ++# define MB_LEN_MAX 16 ++#endif ++ ++/* Some systems, like BeOS, have multibyte encodings but lack mbstate_t. */ ++#if HAVE_MBRTOWC && defined mbstate_t ++# define mbrtowc(pwc, s, n, ps) (mbrtowc) (pwc, s, n, 0) ++#endif ++ + /* The official name of this program (e.g., no `g' prefix). */ + #define PROGRAM_NAME "unexpand" + +@@ -110,6 +127,208 @@ + {NULL, 0, NULL, 0} + }; + ++static FILE *next_file (FILE *fp); ++ ++#if HAVE_MBRTOWC ++static void ++unexpand_multibyte (void) ++{ ++ FILE *fp; /* Input stream. */ ++ mbstate_t i_state; /* Current shift state of the input stream. */ ++ mbstate_t i_state_bak; /* Back up the I_STATE. */ ++ mbstate_t o_state; /* Current shift state of the output stream. */ ++ char buf[MB_LEN_MAX + BUFSIZ]; /* For spooling a read byte sequence. */ ++ char *bufpos; /* Next read position of BUF. */ ++ size_t buflen = 0; /* The length of the byte sequence in buf. */ ++ wint_t wc; /* A gotten wide character. */ ++ size_t mblength; /* The byte size of a multibyte character ++ which shows as same character as WC. */ ++ ++ /* Index in `tab_list' of next tabstop: */ ++ int tab_index = 0; /* For calculating width of pending tabs. */ ++ int print_tab_index = 0; /* For printing as many tabs as possible. */ ++ unsigned int column = 0; /* Column on screen of next char. */ ++ int next_tab_column; /* Column the next tab stop is on. */ ++ int convert = 1; /* If nonzero, perform translations. */ ++ unsigned int pending = 0; /* Pending columns of blanks. */ ++ ++ fp = next_file ((FILE *) NULL); ++ if (fp == NULL) ++ return; ++ ++ memset (&o_state, '\0', sizeof(mbstate_t)); ++ memset (&i_state, '\0', sizeof(mbstate_t)); ++ ++ for (;;) ++ { ++ if (buflen < MB_LEN_MAX && !feof(fp) && !ferror(fp)) ++ { ++ memmove (buf, bufpos, buflen); ++ buflen += fread (buf + buflen, sizeof(char), BUFSIZ, fp); ++ bufpos = buf; ++ } ++ ++ /* Get a wide character. */ ++ if (buflen < 1) ++ { ++ mblength = 1; ++ wc = WEOF; ++ } ++ else ++ { ++ i_state_bak = i_state; ++ mblength = mbrtowc ((wchar_t *)&wc, bufpos, buflen, &i_state); ++ } ++ ++ if (mblength == (size_t)-1 || mblength == (size_t)-2) ++ { ++ i_state = i_state_bak; ++ wc = L'\0'; ++ } ++ ++ if (wc == L' ' && convert && column < INT_MAX) ++ { ++ ++pending; ++ ++column; ++ } ++ else if (wc == L'\t' && convert) ++ { ++ if (tab_size == 0) ++ { ++ /* Do not let tab_index == first_free_tab; ++ stop when it is 1 less. */ ++ while (tab_index < first_free_tab - 1 ++ && column >= tab_list[tab_index]) ++ tab_index++; ++ next_tab_column = tab_list[tab_index]; ++ if (tab_index < first_free_tab - 1) ++ tab_index++; ++ if (column >= next_tab_column) ++ { ++ convert = 0; /* Ran out of tab stops. */ ++ goto flush_pend_mb; ++ } ++ } ++ else ++ { ++ next_tab_column = column + tab_size - column % tab_size; ++ } ++ pending += next_tab_column - column; ++ column = next_tab_column; ++ } ++ else ++ { ++flush_pend_mb: ++ /* Flush pending spaces. Print as many tabs as possible, ++ then print the rest as spaces. */ ++ if (pending == 1) ++ { ++ putchar (' '); ++ pending = 0; ++ } ++ column -= pending; ++ while (pending > 0) ++ { ++ if (tab_size == 0) ++ { ++ /* Do not let print_tab_index == first_free_tab; ++ stop when it is 1 less. */ ++ while (print_tab_index < first_free_tab - 1 ++ && column >= tab_list[print_tab_index]) ++ print_tab_index++; ++ next_tab_column = tab_list[print_tab_index]; ++ if (print_tab_index < first_free_tab - 1) ++ print_tab_index++; ++ } ++ else ++ { ++ next_tab_column = ++ column + tab_size - column % tab_size; ++ } ++ if (next_tab_column - column <= pending) ++ { ++ putchar ('\t'); ++ pending -= next_tab_column - column; ++ column = next_tab_column; ++ } ++ else ++ { ++ --print_tab_index; ++ column += pending; ++ while (pending != 0) ++ { ++ putchar (' '); ++ pending--; ++ } ++ } ++ } ++ ++ if (wc == WEOF) ++ { ++ fp = next_file (fp); ++ if (fp == NULL) ++ break; /* No more files. */ ++ else ++ { ++ memset (&i_state, '\0', sizeof(mbstate_t)); ++ continue; ++ } ++ } ++ ++ if (mblength == (size_t)-1 || mblength == (size_t)-2) ++ { ++ if (convert) ++ { ++ ++column; ++ if (convert_entire_line == 0) ++ convert = 0; ++ } ++ mblength = 1; ++ putchar (buf[0]); ++ } ++ else if (mblength == 0) ++ { ++ if (convert && convert_entire_line == 0) ++ convert = 0; ++ mblength = 1; ++ putchar ('\0'); ++ } ++ else ++ { ++ if (convert) ++ { ++ if (wc == L'\b') ++ { ++ if (column > 0) ++ --column; ++ } ++ else ++ { ++ int width; /* The width of WC. */ ++ ++ width = wcwidth (wc); ++ column += (width > 0) ? width : 0; ++ if (convert_entire_line == 0) ++ convert = 0; ++ } ++ } ++ ++ if (wc == L'\n') ++ { ++ tab_index = print_tab_index = 0; ++ column = pending = 0; ++ convert = 1; ++ } ++ fwrite (bufpos, sizeof(char), mblength, stdout); ++ } ++ } ++ buflen -= mblength; ++ bufpos += mblength; ++ } ++} ++#endif ++ ++ + void + usage (int status) + { +@@ -532,7 +751,12 @@ + + file_list = (optind < argc ? &argv[optind] : stdin_argv); + +- unexpand (); ++#if HAVE_MBRTOWC ++ if (MB_CUR_MAX > 1) ++ unexpand_multibyte (); ++ else ++#endif ++ unexpand (); + + if (have_read_stdin && fclose (stdin) != 0) + error (EXIT_FAILURE, errno, "-"); +--- coreutils-5.95/src/fold.c.i18n 2005-08-12 08:29:38.000000000 +0100 ++++ coreutils-5.95/src/fold.c 2006-05-15 15:08:57.000000000 +0100 +@@ -23,11 +23,33 @@ + #include + #include + ++/* Get mbstate_t, mbrtowc(), wcwidth(). */ ++#if HAVE_WCHAR_H ++# include ++#endif ++ ++/* Get iswprint(), iswblank(), wcwidth(). */ ++#if HAVE_WCTYPE_H ++# include ++#endif ++ + #include "system.h" + #include "error.h" + #include "quote.h" + #include "xstrtol.h" + ++/* MB_LEN_MAX is incorrectly defined to be 1 in at least one GCC ++ installation; work around this configuration error. */ ++#if !defined MB_LEN_MAX || MB_LEN_MAX < 2 ++# undef MB_LEN_MAX ++# define MB_LEN_MAX 16 ++#endif ++ ++/* Some systems, like BeOS, have multibyte encodings but lack mbstate_t. */ ++#if HAVE_MBRTOWC && defined mbstate_t ++# define mbrtowc(pwc, s, n, ps) (mbrtowc) (pwc, s, n, 0) ++#endif ++ + #define TAB_WIDTH 8 + + /* The official name of this program (e.g., no `g' prefix). */ +@@ -35,23 +57,44 @@ + + #define AUTHORS "David MacKenzie" + ++#define FATAL_ERROR(Message) \ ++ do \ ++ { \ ++ error (0, 0, (Message)); \ ++ usage (2); \ ++ } \ ++ while (0) ++ ++enum operating_mode ++{ ++ /* Fold texts by columns that are at the given positions. */ ++ column_mode, ++ ++ /* Fold texts by bytes that are at the given positions. */ ++ byte_mode, ++ ++ /* Fold texts by characters that are at the given positions. */ ++ character_mode, ++}; ++ + /* The name this program was run with. */ + char *program_name; + ++/* The argument shows current mode. (Default: column_mode) */ ++static enum operating_mode operating_mode; ++ + /* If nonzero, try to break on whitespace. */ + static bool break_spaces; + +-/* If nonzero, count bytes, not column positions. */ +-static bool count_bytes; +- + /* If nonzero, at least one of the files we read was standard input. */ + static bool have_read_stdin; + +-static char const shortopts[] = "bsw:0::1::2::3::4::5::6::7::8::9::"; ++static char const shortopts[] = "bcsw:0::1::2::3::4::5::6::7::8::9::"; + + static struct option const longopts[] = + { + {"bytes", no_argument, NULL, 'b'}, ++ {"characters", no_argument, NULL, 'c'}, + {"spaces", no_argument, NULL, 's'}, + {"width", required_argument, NULL, 'w'}, + {GETOPT_HELP_OPTION_DECL}, +@@ -81,6 +124,7 @@ + "), stdout); + fputs (_("\ + -b, --bytes count bytes rather than columns\n\ ++ -c, --characters count characters rather than columns\n\ + -s, --spaces break at spaces\n\ + -w, --width=WIDTH use WIDTH columns instead of 80\n\ + "), stdout); +@@ -98,7 +142,7 @@ + static size_t + adjust_column (size_t column, char c) + { +- if (!count_bytes) ++ if (operating_mode != byte_mode) + { + if (c == '\b') + { +@@ -117,35 +161,14 @@ + return column; + } + +-/* Fold file FILENAME, or standard input if FILENAME is "-", +- to stdout, with maximum line length WIDTH. +- Return true if successful. */ +- +-static bool +-fold_file (char *filename, size_t width) ++static void ++fold_text (FILE *istream, size_t width, int *saved_errno) + { +- FILE *istream; + int c; + size_t column = 0; /* Screen column where next char will go. */ + size_t offset_out = 0; /* Index in `line_out' for next char. */ + static char *line_out = NULL; + static size_t allocated_out = 0; +- int saved_errno; +- +- if (STREQ (filename, "-")) +- { +- istream = stdin; +- have_read_stdin = true; +- } +- else +- istream = fopen (filename, "r"); +- +- if (istream == NULL) +- { +- error (0, errno, "%s", filename); +- return false; +- } +- + while ((c = getc (istream)) != EOF) + { + if (offset_out + 1 >= allocated_out) +@@ -172,6 +195,15 @@ + bool found_blank = false; + size_t logical_end = offset_out; + ++ /* If LINE_OUT has no wide character, ++ put a new wide character in LINE_OUT ++ if column is bigger than width. */ ++ if (offset_out == 0) ++ { ++ line_out[offset_out++] = c; ++ continue; ++ } ++ + /* Look for the last blank. */ + while (logical_end) + { +@@ -218,11 +250,225 @@ + line_out[offset_out++] = c; + } + +- saved_errno = errno; ++ *saved_errno = errno; ++ ++ if (offset_out) ++ fwrite (line_out, sizeof (char), (size_t) offset_out, stdout); ++ ++ free(line_out); ++} ++ ++#if HAVE_MBRTOWC ++static void ++fold_multibyte_text (FILE *istream, int width, int *saved_errno) ++{ ++ char buf[MB_LEN_MAX + BUFSIZ]; /* For spooling a read byte sequence. */ ++ size_t buflen = 0; /* The length of the byte sequence in buf. */ ++ char *bufpos; /* Next read position of BUF. */ ++ wint_t wc; /* A gotten wide character. */ ++ size_t mblength; /* The byte size of a multibyte character which shows ++ as same character as WC. */ ++ mbstate_t state, state_bak; /* State of the stream. */ ++ int convfail; /* 1, when conversion is failed. Otherwise 0. */ ++ ++ char *line_out = NULL; ++ size_t offset_out = 0; /* Index in `line_out' for next char. */ ++ size_t allocated_out = 0; ++ ++ int increment; ++ size_t column = 0; ++ ++ size_t last_blank_pos; ++ size_t last_blank_column; ++ int is_blank_seen; ++ int last_blank_increment; ++ int is_bs_following_last_blank; ++ size_t bs_following_last_blank_num; ++ int is_cr_after_last_blank; ++ ++#define CLEAR_FLAGS \ ++ do \ ++ { \ ++ last_blank_pos = 0; \ ++ last_blank_column = 0; \ ++ is_blank_seen = 0; \ ++ is_bs_following_last_blank = 0; \ ++ bs_following_last_blank_num = 0; \ ++ is_cr_after_last_blank = 0; \ ++ } \ ++ while (0) ++ ++#define START_NEW_LINE \ ++ do \ ++ { \ ++ putchar ('\n'); \ ++ column = 0; \ ++ offset_out = 0; \ ++ CLEAR_FLAGS; \ ++ } \ ++ while (0) ++ ++ CLEAR_FLAGS; ++ memset (&state, '\0', sizeof(mbstate_t)); ++ ++ for (;; bufpos += mblength, buflen -= mblength) ++ { ++ if (buflen < MB_LEN_MAX && !feof (istream) && !ferror (istream)) ++ { ++ memmove (buf, bufpos, buflen); ++ buflen += fread (buf + buflen, sizeof(char), BUFSIZ, istream); ++ bufpos = buf; ++ } ++ ++ if (buflen < 1) ++ break; ++ ++ /* Get a wide character. */ ++ convfail = 0; ++ state_bak = state; ++ mblength = mbrtowc ((wchar_t *)&wc, bufpos, buflen, &state); ++ ++ switch (mblength) ++ { ++ case (size_t)-1: ++ case (size_t)-2: ++ convfail++; ++ state = state_bak; ++ /* Fall through. */ ++ ++ case 0: ++ mblength = 1; ++ break; ++ } ++ ++rescan: ++ if (operating_mode == byte_mode) /* byte mode */ ++ increment = mblength; ++ else if (operating_mode == character_mode) /* character mode */ ++ increment = 1; ++ else /* column mode */ ++ { ++ if (convfail) ++ increment = 1; ++ else ++ { ++ switch (wc) ++ { ++ case L'\n': ++ fwrite (line_out, sizeof(char), offset_out, stdout); ++ START_NEW_LINE; ++ continue; ++ ++ case L'\b': ++ increment = (column > 0) ? -1 : 0; ++ break; ++ ++ case L'\r': ++ increment = -1 * column; ++ break; ++ ++ case L'\t': ++ increment = 8 - column % 8; ++ break; ++ ++ default: ++ increment = wcwidth (wc); ++ increment = (increment < 0) ? 0 : increment; ++ } ++ } ++ } ++ ++ if (column + increment > width && break_spaces && last_blank_pos) ++ { ++ fwrite (line_out, sizeof(char), last_blank_pos, stdout); ++ putchar ('\n'); ++ ++ offset_out = offset_out - last_blank_pos; ++ column = column - last_blank_column + ((is_cr_after_last_blank) ++ ? last_blank_increment : bs_following_last_blank_num); ++ memmove (line_out, line_out + last_blank_pos, offset_out); ++ CLEAR_FLAGS; ++ goto rescan; ++ } ++ ++ if (column + increment > width && column != 0) ++ { ++ fwrite (line_out, sizeof(char), offset_out, stdout); ++ START_NEW_LINE; ++ goto rescan; ++ } ++ ++ if (allocated_out < offset_out + mblength) ++ { ++ allocated_out += 1024; ++ line_out = xrealloc (line_out, allocated_out); ++ } ++ ++ memcpy (line_out + offset_out, bufpos, mblength); ++ offset_out += mblength; ++ column += increment; ++ ++ if (is_blank_seen && !convfail && wc == L'\r') ++ is_cr_after_last_blank = 1; ++ ++ if (is_bs_following_last_blank && !convfail && wc == L'\b') ++ ++bs_following_last_blank_num; ++ else ++ is_bs_following_last_blank = 0; ++ ++ if (break_spaces && !convfail && iswblank (wc)) ++ { ++ last_blank_pos = offset_out; ++ last_blank_column = column; ++ is_blank_seen = 1; ++ last_blank_increment = increment; ++ is_bs_following_last_blank = 1; ++ bs_following_last_blank_num = 0; ++ is_cr_after_last_blank = 0; ++ } ++ } ++ ++ *saved_errno = errno; + + if (offset_out) + fwrite (line_out, sizeof (char), (size_t) offset_out, stdout); + ++ free(line_out); ++} ++#endif ++ ++/* Fold file FILENAME, or standard input if FILENAME is "-", ++ to stdout, with maximum line length WIDTH. ++ Return 0 if successful, 1 if an error occurs. */ ++ ++static int ++fold_file (char *filename, int width) ++{ ++ FILE *istream; ++ int saved_errno; ++ ++ if (STREQ (filename, "-")) ++ { ++ istream = stdin; ++ have_read_stdin = 1; ++ } ++ else ++ istream = fopen (filename, "r"); ++ ++ if (istream == NULL) ++ { ++ error (0, errno, "%s", filename); ++ return 1; ++ } ++ ++ /* Define how ISTREAM is being folded. */ ++#if HAVE_MBRTOWC ++ if (MB_CUR_MAX > 1) ++ fold_multibyte_text (istream, width, &saved_errno); ++ else ++#endif ++ fold_text (istream, width, &saved_errno); ++ + if (ferror (istream)) + { + error (0, saved_errno, "%s", filename); +@@ -255,7 +501,8 @@ + + atexit (close_stdout); + +- break_spaces = count_bytes = have_read_stdin = false; ++ operating_mode = column_mode; ++ break_spaces = have_read_stdin = false; + + while ((optc = getopt_long (argc, argv, shortopts, longopts, NULL)) != -1) + { +@@ -264,7 +511,15 @@ + switch (optc) + { + case 'b': /* Count bytes rather than columns. */ +- count_bytes = true; ++ if (operating_mode != column_mode) ++ FATAL_ERROR (_("only one way of folding may be specified")); ++ operating_mode = byte_mode; ++ break; ++ ++ case 'c': ++ if (operating_mode != column_mode) ++ FATAL_ERROR (_("only one way of folding may be specified")); ++ operating_mode = character_mode; + break; + + case 's': /* Break at word boundaries. */ +--- coreutils-5.95/src/cut.c.i18n 2005-08-12 08:16:25.000000000 +0100 ++++ coreutils-5.95/src/cut.c 2006-05-15 15:08:57.000000000 +0100 +@@ -29,6 +29,11 @@ + #include + #include + #include ++ ++/* Get mbstate_t, mbrtowc(). */ ++#if HAVE_WCHAR_H ++# include ++#endif + #include "system.h" + + #include "error.h" +@@ -37,6 +42,18 @@ + #include "quote.h" + #include "xstrndup.h" + ++/* MB_LEN_MAX is incorrectly defined to be 1 in at least one GCC ++ installation; work around this configuration error. */ ++#if !defined MB_LEN_MAX || MB_LEN_MAX < 2 ++# undef MB_LEN_MAX ++# define MB_LEN_MAX 16 ++#endif ++ ++/* Some systems, like BeOS, have multibyte encodings but lack mbstate_t. */ ++#if HAVE_MBRTOWC && defined mbstate_t ++# define mbrtowc(pwc, s, n, ps) (mbrtowc) (pwc, s, n, 0) ++#endif ++ + /* The official name of this program (e.g., no `g' prefix). */ + #define PROGRAM_NAME "cut" + +@@ -67,6 +84,52 @@ + } \ + while (0) + ++/* Refill the buffer BUF to get a multibyte character. */ ++#define REFILL_BUFFER(BUF, BUFPOS, BUFLEN, STREAM) \ ++ do \ ++ { \ ++ if (BUFLEN < MB_LEN_MAX && !feof (STREAM) && !ferror (STREAM)) \ ++ { \ ++ memmove (BUF, BUFPOS, BUFLEN); \ ++ BUFLEN += fread (BUF + BUFLEN, sizeof(char), BUFSIZ, STREAM); \ ++ BUFPOS = BUF; \ ++ } \ ++ } \ ++ while (0) ++ ++/* Get wide character on BUFPOS. BUFPOS is not included after that. ++ If byte sequence is not valid as a character, CONVFAIL is 1. Otherwise 0. */ ++#define GET_NEXT_WC_FROM_BUFFER(WC, BUFPOS, BUFLEN, MBLENGTH, STATE, CONVFAIL) \ ++ do \ ++ { \ ++ mbstate_t state_bak; \ ++ \ ++ if (BUFLEN < 1) \ ++ { \ ++ WC = WEOF; \ ++ break; \ ++ } \ ++ \ ++ /* Get a wide character. */ \ ++ CONVFAIL = 0; \ ++ state_bak = STATE; \ ++ MBLENGTH = mbrtowc ((wchar_t *)&WC, BUFPOS, BUFLEN, &STATE); \ ++ \ ++ switch (MBLENGTH) \ ++ { \ ++ case (size_t)-1: \ ++ case (size_t)-2: \ ++ CONVFAIL++; \ ++ STATE = state_bak; \ ++ /* Fall througn. */ \ ++ \ ++ case 0: \ ++ MBLENGTH = 1; \ ++ break; \ ++ } \ ++ } \ ++ while (0) ++ + struct range_pair + { + size_t lo; +@@ -85,7 +148,7 @@ + /* The number of bytes allocated for FIELD_1_BUFFER. */ + static size_t field_1_bufsize; + +-/* The largest field or byte index used as an endpoint of a closed ++/* The largest byte, character or field index used as an endpoint of a closed + or degenerate range specification; this doesn't include the starting + index of right-open-ended ranges. For example, with either range spec + `2-5,9-', `2-3,5,9-' this variable would be set to 5. */ +@@ -97,10 +160,11 @@ + + /* This is a bit vector. + In byte mode, which bytes to output. ++ In character mode, which characters to output. + In field mode, which DELIM-separated fields to output. +- Both bytes and fields are numbered starting with 1, ++ Bytes, characters and fields are numbered starting with 1, + so the zeroth bit of this array is unused. +- A field or byte K has been selected if ++ A byte, character or field K has been selected if + (K <= MAX_RANGE_ENDPOINT and is_printable_field(K)) + || (EOL_RANGE_START > 0 && K >= EOL_RANGE_START). */ + static unsigned char *printable_field; +@@ -109,9 +173,12 @@ + { + undefined_mode, + +- /* Output characters that are in the given bytes. */ ++ /* Output bytes that are at the given positions. */ + byte_mode, + ++ /* Output characters that are at the given positions. */ ++ character_mode, ++ + /* Output the given delimeter-separated fields. */ + field_mode + }; +@@ -121,6 +188,13 @@ + + static enum operating_mode operating_mode; + ++/* If nonzero, when in byte mode, don't split multibyte characters. */ ++static int byte_mode_character_aware; ++ ++/* If nonzero, the function for single byte locale is work ++ if this program runs on multibyte locale. */ ++static int force_singlebyte_mode; ++ + /* If true do not output lines containing no delimeter characters. + Otherwise, all such lines are printed. This option is valid only + with field mode. */ +@@ -132,6 +206,9 @@ + + /* The delimeter character for field mode. */ + static unsigned char delim; ++#if HAVE_WCHAR_H ++static wchar_t wcdelim; ++#endif + + /* True if the --output-delimiter=STRING option was specified. */ + static bool output_delimiter_specified; +@@ -205,7 +282,7 @@ + -f, --fields=LIST select only these fields; also print any line\n\ + that contains no delimiter character, unless\n\ + the -s option is specified\n\ +- -n (ignored)\n\ ++ -n with -b: don't split multibyte characters\n\ + "), stdout); + fputs (_("\ + --complement complement the set of selected bytes, characters\n\ +@@ -360,7 +437,7 @@ + in_digits = false; + /* Starting a range. */ + if (dash_found) +- FATAL_ERROR (_("invalid byte or field list")); ++ FATAL_ERROR (_("invalid byte, character or field list")); + dash_found = true; + fieldstr++; + +@@ -385,14 +462,16 @@ + if (value == 0) + { + /* `n-'. From `initial' to end of line. */ +- eol_range_start = initial; ++ if (eol_range_start == 0 || ++ (eol_range_start != 0 && eol_range_start > initial)) ++ eol_range_start = initial; + field_found = true; + } + else + { + /* `m-n' or `-n' (1-n). */ + if (value < initial) +- FATAL_ERROR (_("invalid byte or field list")); ++ FATAL_ERROR (_("invalid byte, character or field list")); + + /* Is there already a range going to end of line? */ + if (eol_range_start != 0) +@@ -465,6 +544,9 @@ + if (operating_mode == byte_mode) + error (0, 0, + _("byte offset %s is too large"), quote (bad_num)); ++ else if (operating_mode == character_mode) ++ error (0, 0, ++ _("character offset %s is too large"), quote (bad_num)); + else + error (0, 0, + _("field number %s is too large"), quote (bad_num)); +@@ -475,7 +557,7 @@ + fieldstr++; + } + else +- FATAL_ERROR (_("invalid byte or field list")); ++ FATAL_ERROR (_("invalid byte, character or field list")); + } + + max_range_endpoint = 0; +@@ -568,6 +650,63 @@ + } + } + ++#if HAVE_MBRTOWC ++/* This function is in use for the following case. ++ ++ 1. Read from the stream STREAM, printing to standard output any selected ++ characters. ++ ++ 2. Read from stream STREAM, printing to standard output any selected bytes, ++ without splitting multibyte characters. */ ++ ++static void ++cut_characters_or_cut_bytes_no_split (FILE *stream) ++{ ++ int idx; /* number of bytes or characters in the line so far. */ ++ char buf[MB_LEN_MAX + BUFSIZ]; /* For spooling a read byte sequence. */ ++ char *bufpos; /* Next read position of BUF. */ ++ size_t buflen; /* The length of the byte sequence in buf. */ ++ wint_t wc; /* A gotten wide character. */ ++ size_t mblength; /* The byte size of a multibyte character which shows ++ as same character as WC. */ ++ mbstate_t state; /* State of the stream. */ ++ int convfail; /* 1, when conversion is failed. Otherwise 0. */ ++ ++ idx = 0; ++ buflen = 0; ++ bufpos = buf; ++ memset (&state, '\0', sizeof(mbstate_t)); ++ ++ while (1) ++ { ++ REFILL_BUFFER (buf, bufpos, buflen, stream); ++ ++ GET_NEXT_WC_FROM_BUFFER (wc, bufpos, buflen, mblength, state, convfail); ++ ++ if (wc == WEOF) ++ { ++ if (idx > 0) ++ putchar ('\n'); ++ break; ++ } ++ else if (wc == L'\n') ++ { ++ putchar ('\n'); ++ idx = 0; ++ } ++ else ++ { ++ idx += (operating_mode == byte_mode) ? mblength : 1; ++ if (print_kth (idx, NULL)) ++ fwrite (bufpos, mblength, sizeof(char), stdout); ++ } ++ ++ buflen -= mblength; ++ bufpos += mblength; ++ } ++} ++#endif ++ + /* Read from stream STREAM, printing to standard output any selected fields. */ + + static void +@@ -689,13 +828,192 @@ + } + } + ++#if HAVE_MBRTOWC ++static void ++cut_fields_mb (FILE *stream) ++{ ++ int c; ++ unsigned int field_idx; ++ int found_any_selected_field; ++ int buffer_first_field; ++ int empty_input; ++ char buf[MB_LEN_MAX + BUFSIZ]; /* For spooling a read byte sequence. */ ++ char *bufpos; /* Next read position of BUF. */ ++ size_t buflen; /* The length of the byte sequence in buf. */ ++ wint_t wc = 0; /* A gotten wide character. */ ++ size_t mblength; /* The byte size of a multibyte character which shows ++ as same character as WC. */ ++ mbstate_t state; /* State of the stream. */ ++ int convfail; /* 1, when conversion is failed. Otherwise 0. */ ++ ++ found_any_selected_field = 0; ++ field_idx = 1; ++ bufpos = buf; ++ buflen = 0; ++ memset (&state, '\0', sizeof(mbstate_t)); ++ ++ c = getc (stream); ++ empty_input = (c == EOF); ++ if (c != EOF) ++ ungetc (c, stream); ++ else ++ wc = WEOF; ++ ++ /* To support the semantics of the -s flag, we may have to buffer ++ all of the first field to determine whether it is `delimited.' ++ But that is unnecessary if all non-delimited lines must be printed ++ and the first field has been selected, or if non-delimited lines ++ must be suppressed and the first field has *not* been selected. ++ That is because a non-delimited line has exactly one field. */ ++ buffer_first_field = (suppress_non_delimited ^ !print_kth (1, NULL)); ++ ++ while (1) ++ { ++ if (field_idx == 1 && buffer_first_field) ++ { ++ int len = 0; ++ ++ while (1) ++ { ++ REFILL_BUFFER (buf, bufpos, buflen, stream); ++ ++ GET_NEXT_WC_FROM_BUFFER ++ (wc, bufpos, buflen, mblength, state, convfail); ++ ++ if (wc == WEOF) ++ break; ++ ++ field_1_buffer = xrealloc (field_1_buffer, len + mblength); ++ memcpy (field_1_buffer + len, bufpos, mblength); ++ len += mblength; ++ buflen -= mblength; ++ bufpos += mblength; ++ ++ if (!convfail && (wc == L'\n' || wc == wcdelim)) ++ break; ++ } ++ ++ if (wc == WEOF) ++ break; ++ ++ /* If the first field extends to the end of line (it is not ++ delimited) and we are printing all non-delimited lines, ++ print this one. */ ++ if (convfail || (!convfail && wc != wcdelim)) ++ { ++ if (suppress_non_delimited) ++ { ++ /* Empty. */ ++ } ++ else ++ { ++ fwrite (field_1_buffer, sizeof (char), len, stdout); ++ /* Make sure the output line is newline terminated. */ ++ if (convfail || (!convfail && wc != L'\n')) ++ putchar ('\n'); ++ } ++ continue; ++ } ++ ++ if (print_kth (1, NULL)) ++ { ++ /* Print the field, but not the trailing delimiter. */ ++ fwrite (field_1_buffer, sizeof (char), len - 1, stdout); ++ found_any_selected_field = 1; ++ } ++ ++field_idx; ++ } ++ ++ if (wc != WEOF) ++ { ++ if (print_kth (field_idx, NULL)) ++ { ++ if (found_any_selected_field) ++ { ++ fwrite (output_delimiter_string, sizeof (char), ++ output_delimiter_length, stdout); ++ } ++ found_any_selected_field = 1; ++ } ++ ++ while (1) ++ { ++ REFILL_BUFFER (buf, bufpos, buflen, stream); ++ ++ GET_NEXT_WC_FROM_BUFFER ++ (wc, bufpos, buflen, mblength, state, convfail); ++ ++ if (wc == WEOF) ++ break; ++ else if (!convfail && (wc == wcdelim || wc == L'\n')) ++ { ++ buflen -= mblength; ++ bufpos += mblength; ++ break; ++ } ++ ++ if (print_kth (field_idx, NULL)) ++ fwrite (bufpos, mblength, sizeof(char), stdout); ++ ++ buflen -= mblength; ++ bufpos += mblength; ++ } ++ } ++ ++ if ((!convfail || wc == L'\n') && buflen < 1) ++ wc = WEOF; ++ ++ if (!convfail && wc == wcdelim) ++ ++field_idx; ++ else if (wc == WEOF || (!convfail && wc == L'\n')) ++ { ++ if (found_any_selected_field ++ || (!empty_input && !(suppress_non_delimited && field_idx == 1))) ++ putchar ('\n'); ++ if (wc == WEOF) ++ break; ++ field_idx = 1; ++ found_any_selected_field = 0; ++ } ++ } ++} ++#endif ++ + static void + cut_stream (FILE *stream) + { +- if (operating_mode == byte_mode) +- cut_bytes (stream); ++#if HAVE_MBRTOWC ++ if (MB_CUR_MAX > 1 && !force_singlebyte_mode) ++ { ++ switch (operating_mode) ++ { ++ case byte_mode: ++ if (byte_mode_character_aware) ++ cut_characters_or_cut_bytes_no_split (stream); ++ else ++ cut_bytes (stream); ++ break; ++ ++ case character_mode: ++ cut_characters_or_cut_bytes_no_split (stream); ++ break; ++ ++ case field_mode: ++ cut_fields_mb (stream); ++ break; ++ ++ default: ++ abort (); ++ } ++ } + else +- cut_fields (stream); ++#endif ++ { ++ if (operating_mode == field_mode) ++ cut_fields (stream); ++ else ++ cut_bytes (stream); ++ } + } + + /* Process file FILE to standard output. +@@ -745,6 +1063,8 @@ + bool ok; + bool delim_specified = false; + char *spec_list_string IF_LINT(= NULL); ++ char mbdelim[MB_LEN_MAX + 1]; ++ size_t delimlen = 0; + + initialize_main (&argc, &argv); + program_name = argv[0]; +@@ -767,7 +1087,6 @@ + switch (optc) + { + case 'b': +- case 'c': + /* Build the byte list. */ + if (operating_mode != undefined_mode) + FATAL_ERROR (_("only one type of list may be specified")); +@@ -775,6 +1094,14 @@ + spec_list_string = optarg; + break; + ++ case 'c': ++ /* Build the character list. */ ++ if (operating_mode != undefined_mode) ++ FATAL_ERROR (_("only one type of list may be specified")); ++ operating_mode = character_mode; ++ spec_list_string = optarg; ++ break; ++ + case 'f': + /* Build the field list. */ + if (operating_mode != undefined_mode) +@@ -786,10 +1113,35 @@ + case 'd': + /* New delimiter. */ + /* Interpret -d '' to mean `use the NUL byte as the delimiter.' */ +- if (optarg[0] != '\0' && optarg[1] != '\0') +- FATAL_ERROR (_("the delimiter must be a single character")); +- delim = optarg[0]; +- delim_specified = true; ++#if HAVE_MBRTOWC ++ { ++ if(MB_CUR_MAX > 1) ++ { ++ mbstate_t state; ++ ++ memset (&state, '\0', sizeof(mbstate_t)); ++ delimlen = mbrtowc (&wcdelim, optarg, strnlen(optarg, MB_LEN_MAX), &state); ++ ++ if (delimlen == (size_t)-1 || delimlen == (size_t)-2) ++ ++force_singlebyte_mode; ++ else ++ { ++ delimlen = (delimlen < 1) ? 1 : delimlen; ++ if (wcdelim != L'\0' && *(optarg + delimlen) != '\0') ++ FATAL_ERROR (_("the delimiter must be a single character")); ++ memcpy (mbdelim, optarg, delimlen); ++ } ++ } ++ ++ if (MB_CUR_MAX <= 1 || force_singlebyte_mode) ++#endif ++ { ++ if (optarg[0] != '\0' && optarg[1] != '\0') ++ FATAL_ERROR (_("the delimiter must be a single character")); ++ delim = (unsigned char) optarg[0]; ++ } ++ delim_specified = true; ++ } + break; + + case OUTPUT_DELIMITER_OPTION: +@@ -802,6 +1154,7 @@ + break; + + case 'n': ++ byte_mode_character_aware = 1; + break; + + case 's': +@@ -824,7 +1177,7 @@ + if (operating_mode == undefined_mode) + FATAL_ERROR (_("you must specify a list of bytes, characters, or fields")); + +- if (delim != '\0' && operating_mode != field_mode) ++ if (delim_specified && operating_mode != field_mode) + FATAL_ERROR (_("an input delimiter may be specified only\ + when operating on fields")); + +@@ -851,15 +1204,34 @@ + } + + if (!delim_specified) +- delim = '\t'; ++ { ++ delim = '\t'; ++#ifdef HAVE_MBRTOWC ++ wcdelim = L'\t'; ++ mbdelim[0] = '\t'; ++ mbdelim[1] = '\0'; ++ delimlen = 1; ++#endif ++ } + + if (output_delimiter_string == NULL) + { +- static char dummy[2]; +- dummy[0] = delim; +- dummy[1] = '\0'; +- output_delimiter_string = dummy; +- output_delimiter_length = 1; ++#ifdef HAVE_MBRTOWC ++ if (MB_CUR_MAX > 1 && !force_singlebyte_mode) ++ { ++ output_delimiter_string = xstrdup(mbdelim); ++ output_delimiter_length = delimlen; ++ } ++ ++ if (MB_CUR_MAX <= 1 || force_singlebyte_mode) ++#endif ++ { ++ static char dummy[2]; ++ dummy[0] = delim; ++ dummy[1] = '\0'; ++ output_delimiter_string = dummy; ++ output_delimiter_length = 1; ++ } + } + + if (optind == argc) +--- coreutils-5.95/tests/sort/Makefile.am.i18n 2005-10-24 22:02:25.000000000 +0100 ++++ coreutils-5.95/tests/sort/Makefile.am 2006-05-15 15:08:57.000000000 +0100 +@@ -43,14 +43,16 @@ + nul-nls.E use-nl.O use-nl.E o2.O o2.E nul-tab.O nul-tab.E + ##test-files-end + +-EXTRA_DIST = Test.pm $x-tests $(explicit) $(maint_gen) +-noinst_SCRIPTS = $x-tests ++run_gen += mb1.O mb2.O ++ ++EXTRA_DIST = Test.pm $x-tests $(explicit) $(maint_gen) mb1.I mb1.X mb2.I mb2.X ++noinst_SCRIPTS = $x-tests # $x-mb-tests + TESTS_ENVIRONMENT = \ + PATH="`pwd`/../../src$(PATH_SEPARATOR)$$PATH" + + editpl = sed -e 's,@''PERL''@,$(PERL),g' -e 's,@''srcdir''@,$(srcdir),g' + +-TESTS = $x-tests ++TESTS = $x-tests $x-mb-tests + + mk_script = $(srcdir)/../mk-script + $(srcdir)/$x-tests: $(mk_script) Test.pm Makefile.am +--- /dev/null 2006-05-15 09:11:19.652773000 +0100 ++++ coreutils-5.95/tests/sort/mb1.I 2006-05-15 15:08:57.000000000 +0100 +@@ -0,0 +1,4 @@ ++Apple＠10 ++Banana＠5 ++Citrus＠20 ++Cherry＠30 +--- /dev/null 2006-05-15 09:11:19.652773000 +0100 ++++ coreutils-5.95/tests/sort/mb1.X 2006-05-15 15:08:57.000000000 +0100 +@@ -0,0 +1,4 @@ ++Banana＠5 ++Apple＠10 ++Citrus＠20 ++Cherry＠30 +--- /dev/null 2006-05-15 09:11:19.652773000 +0100 ++++ coreutils-5.95/tests/sort/mb2.X 2006-05-15 15:08:57.000000000 +0100 +@@ -0,0 +1,4 @@ ++Citrus＠ＡＡ20＠＠5 ++Cherry＠ＡＡ30＠＠10 ++Apple＠ＡＡ10＠＠20 ++Banana＠ＡＡ5＠＠30 +--- /dev/null 2006-05-15 09:11:19.652773000 +0100 ++++ coreutils-5.95/tests/sort/sort-mb-tests 2006-05-15 15:08:57.000000000 +0100 @@ -0,0 +1,58 @@ +#! /bin/sh +case $# in @@ -3971,58 +3991,8 @@ +fi +test $errors = 0 || errors=1 +exit $errors ---- /dev/null 2005-10-10 09:36:06.437701000 +0100 -+++ coreutils-5.93/tests/sort/mb1.I 2005-12-23 08:53:01.000000000 +0000 -@@ -0,0 +1,4 @@ -+Apple＠10 -+Banana＠5 -+Citrus＠20 -+Cherry＠30 ---- /dev/null 2005-10-10 09:36:06.437701000 +0100 -+++ coreutils-5.93/tests/sort/mb2.I 2005-12-23 08:53:01.000000000 +0000 -@@ -0,0 +1,4 @@ -+Apple＠ＡＡ10＠＠20 -+Banana＠ＡＡ5＠＠30 -+Citrus＠ＡＡ20＠＠5 -+Cherry＠ＡＡ30＠＠10 ---- /dev/null 2005-10-10 09:36:06.437701000 +0100 -+++ coreutils-5.93/tests/sort/mb1.X 2005-12-23 08:53:01.000000000 +0000 -@@ -0,0 +1,4 @@ -+Banana＠5 -+Apple＠10 -+Citrus＠20 -+Cherry＠30 ---- /dev/null 2005-10-10 09:36:06.437701000 +0100 -+++ coreutils-5.93/tests/sort/mb2.X 2005-12-23 08:53:01.000000000 +0000 -@@ -0,0 +1,4 @@ -+Citrus＠ＡＡ20＠＠5 -+Cherry＠ＡＡ30＠＠10 -+Apple＠ＡＡ10＠＠20 -+Banana＠ＡＡ5＠＠30 ---- coreutils-5.93/tests/sort/Makefile.am.i18n 2005-10-24 22:02:25.000000000 +0100 -+++ coreutils-5.93/tests/sort/Makefile.am 2005-12-23 08:53:01.000000000 +0000 -@@ -43,14 +43,16 @@ - nul-nls.E use-nl.O use-nl.E o2.O o2.E nul-tab.O nul-tab.E - ##test-files-end - --EXTRA_DIST = Test.pm $x-tests $(explicit) $(maint_gen) --noinst_SCRIPTS = $x-tests -+run_gen += mb1.O mb2.O -+ -+EXTRA_DIST = Test.pm $x-tests $(explicit) $(maint_gen) mb1.I mb1.X mb2.I mb2.X -+noinst_SCRIPTS = $x-tests # $x-mb-tests - TESTS_ENVIRONMENT = \ - PATH="`pwd`/../../src$(PATH_SEPARATOR)$$PATH" - - editpl = sed -e 's,@''PERL''@,$(PERL),g' -e 's,@''srcdir''@,$(srcdir),g' - --TESTS = $x-tests -+TESTS = $x-tests $x-mb-tests - - mk_script = $(srcdir)/../mk-script - $(srcdir)/$x-tests: $(mk_script) Test.pm Makefile.am ---- coreutils-5.93/tests/sort/Makefile.in.i18n 2005-11-05 22:12:25.000000000 +0000 -+++ coreutils-5.93/tests/sort/Makefile.in 2005-12-23 09:00:37.000000000 +0000 +--- coreutils-5.95/tests/sort/Makefile.in.i18n 2006-05-04 13:52:51.000000000 +0100 ++++ coreutils-5.95/tests/sort/Makefile.in 2006-05-15 15:08:57.000000000 +0100 @@ -107,25 +107,25 @@ $(top_srcdir)/m4/onceonly_2_57.m4 $(top_srcdir)/m4/openat.m4 \ $(top_srcdir)/m4/pathmax.m4 $(top_srcdir)/m4/perl.m4 \ @@ -4093,7 +4063,7 @@ LN_S = @LN_S@ LTLIBICONV = @LTLIBICONV@ LTLIBINTL = @LTLIBINTL@ -@@ -268,30 +270,23 @@ +@@ -266,30 +268,23 @@ build_os = @build_os@ build_vendor = @build_vendor@ datadir = @datadir@ @@ -4124,7 +4094,7 @@ sbindir = @sbindir@ sharedstatedir = @sharedstatedir@ sysconfdir = @sysconfdir@ -@@ -318,33 +313,37 @@ +@@ -316,33 +311,37 @@ neg-nls.I neg-nls.X nul-nls.I nul-nls.X use-nl.I use-nl.X o2.I o2.X nul-tab.I \ nul-tab.X @@ -4185,3 +4155,34 @@ mk_script = $(srcdir)/../mk-script MAINTAINERCLEANFILES = $x-tests $(maint_gen) CLEANFILES = $(run_gen) +--- /dev/null 2006-05-15 09:11:19.652773000 +0100 ++++ coreutils-5.95/tests/sort/mb2.I 2006-05-15 15:08:57.000000000 +0100 +@@ -0,0 +1,4 @@ ++Apple＠ＡＡ10＠＠20 ++Banana＠ＡＡ5＠＠30 ++Citrus＠ＡＡ20＠＠5 ++Cherry＠ＡＡ30＠＠10 +--- coreutils-5.95/lib/linebuffer.h.i18n 2005-05-14 08:58:06.000000000 +0100 ++++ coreutils-5.95/lib/linebuffer.h 2006-05-15 15:08:57.000000000 +0100 +@@ -22,6 +22,11 @@ + + # include + ++/* Get mbstate_t. */ ++# if HAVE_WCHAR_H ++# include ++# endif ++ + /* A `struct linebuffer' holds a line of text. */ + + struct linebuffer +@@ -29,6 +34,9 @@ + size_t size; /* Allocated. */ + size_t length; /* Used. */ + char *buffer; ++# if HAVE_WCHAR_H ++ mbstate_t state; ++# endif + }; + + /* Initialize linebuffer LINEBUFFER for use. */