Description: Remove incorrect strcoll implementation. Author: Carlos O'Donell Origin: git://sourceware.org/glibc.git Bug-RHEL: N/A Bug-Fedora: #1276711 Bug-Upstream: #18589 Upstream status: committed commit 60cf80f09d029257caedc0c8abe7e3e09c64e6c7 Author: Martin Sebor Date: Mon Sep 28 16:55:57 2015 -0400 Let 'make check subdirs=string' succeed even when it's invoked immediately after glibc has been built and before 'make check' (or after 'make clean'). commit facdd9ea29ab94aac2b188ec3cc41f8733d769e0 Author: Carlos O'Donell Date: Fri Oct 9 16:35:53 2015 -0400 Fix typo in bug-strcoll2 (Bug 18589) Fix the copyright year and remove contributed by in the bug-strcoll2 test. In addition add the correct dependency on $(gen-locales) to ensure all the test locales are generated. commit 02018629a1397d03eccceacaf2ee1c50e3c4001c Author: Carlos O'Donell Date: Thu Oct 8 16:54:30 2015 -0400 strcoll: Add bug-strcoll2 to testsuite (Bug 18589). Adds bug-strcoll2 to the string tests, along with the generation of required locales. commit 87701a58e291bd7ac3b407d10a829dac52c9c16e Author: Carlos O'Donell Date: Thu Oct 8 16:34:53 2015 -0400 strcoll: Remove incorrect STRDIFF-based optimization (Bug 18589). The optimization introduced in commit f13c2a8dff2329c6692a80176262ceaaf8a6f74e, causes regressions in sorting for languages that have digraphs that change sort order, like cs_CZ which sorts ch between h and i. My analysis shows the fast-forwarding optimization in STRCOLL advances through a digraph while possibly stopping in the middle which results in a subsequent skipping of the digraph and incorrect sorting. The optimization is incorrect as implemented and because of that I'm removing it for 2.23, and I will also commit this fix for 2.22 where it was originally introduced. This patch reverts the optimization, introduces a new bug-strcoll2.c regression test that tests both cs_CZ.UTF-8 and da_DK.ISO-8859-1 and ensures they sort one digraph each correctly. The optimization can't be applied without regressing this test. Checked on x86_64, bug-strcoll2.c fails without this patch and passes after. This will also get a fix on 2.22 which has the same bug. Index: glibc-2.22/locale/C-collate.c =================================================================== --- glibc-2.22.orig/locale/C-collate.c +++ glibc-2.22/locale/C-collate.c @@ -144,8 +144,6 @@ const struct __locale_data _nl_C_LC_COLL /* _NL_COLLATE_COLLSEQWC */ { .string = (const char *) collseqwc }, /* _NL_COLLATE_CODESET */ - { .string = _nl_C_codeset }, - /* _NL_COLLATE_ENCODING_TYPE */ - { .word = __cet_8bit } + { .string = _nl_C_codeset } } }; Index: glibc-2.22/locale/categories.def =================================================================== --- glibc-2.22.orig/locale/categories.def +++ glibc-2.22/locale/categories.def @@ -58,7 +58,6 @@ DEFINE_CATEGORY DEFINE_ELEMENT (_NL_COLLATE_COLLSEQMB, "collate-collseqmb", std, wstring) DEFINE_ELEMENT (_NL_COLLATE_COLLSEQWC, "collate-collseqwc", std, wstring) DEFINE_ELEMENT (_NL_COLLATE_CODESET, "collate-codeset", std, string) - DEFINE_ELEMENT (_NL_COLLATE_ENCODING_TYPE, "collate-encoding-type", std, word) ), NO_POSTLOAD) Index: glibc-2.22/locale/langinfo.h =================================================================== --- glibc-2.22.orig/locale/langinfo.h +++ glibc-2.22/locale/langinfo.h @@ -255,7 +255,6 @@ enum _NL_COLLATE_COLLSEQMB, _NL_COLLATE_COLLSEQWC, _NL_COLLATE_CODESET, - _NL_COLLATE_ENCODING_TYPE, _NL_NUM_LC_COLLATE, /* LC_CTYPE category: character classification. Index: glibc-2.22/locale/localeinfo.h =================================================================== --- glibc-2.22.orig/locale/localeinfo.h +++ glibc-2.22/locale/localeinfo.h @@ -110,14 +110,6 @@ enum coll_sort_rule sort_mask }; -/* Collation encoding type. */ -enum collation_encoding_type -{ - __cet_other, - __cet_8bit, - __cet_utf8 -}; - /* We can map the types of the entries into a few categories. */ enum value_type { Index: glibc-2.22/locale/programs/ld-collate.c =================================================================== --- glibc-2.22.orig/locale/programs/ld-collate.c +++ glibc-2.22/locale/programs/ld-collate.c @@ -32,7 +32,6 @@ #include "linereader.h" #include "locfile.h" #include "elem-hash.h" -#include "../localeinfo.h" /* Uncomment the following line in the production version. */ /* #define NDEBUG 1 */ @@ -2131,8 +2130,6 @@ collate_output (struct localedef_t *loca /* The words have to be handled specially. */ if (idx == _NL_ITEM_INDEX (_NL_COLLATE_SYMB_HASH_SIZEMB)) add_locale_uint32 (&file, 0); - else if (idx == _NL_ITEM_INDEX (_NL_COLLATE_ENCODING_TYPE)) - add_locale_uint32 (&file, __cet_other); else add_locale_empty (&file); } @@ -2496,12 +2493,6 @@ collate_output (struct localedef_t *loca add_locale_raw_data (&file, collate->mbseqorder, 256); add_locale_collseq_table (&file, &collate->wcseqorder); add_locale_string (&file, charmap->code_set_name); - if (strcmp (charmap->code_set_name, "UTF-8") == 0) - add_locale_uint32 (&file, __cet_utf8); - else if (charmap->mb_cur_max == 1) - add_locale_uint32 (&file, __cet_8bit); - else - add_locale_uint32 (&file, __cet_other); write_locale_data (output_path, LC_COLLATE, "LC_COLLATE", &file); obstack_free (&weightpool, NULL); Index: glibc-2.22/string/Makefile =================================================================== --- glibc-2.22.orig/string/Makefile +++ glibc-2.22/string/Makefile @@ -54,7 +54,7 @@ tests := tester inl-tester noinl-tester tst-strtok tst-strxfrm bug-strcoll1 tst-strfry \ bug-strtok1 $(addprefix test-,$(strop-tests)) \ bug-envz1 tst-strxfrm2 tst-endian tst-svc2 \ - tst-strtok_r + tst-strtok_r bug-strcoll2 xtests = tst-strcoll-overflow @@ -75,4 +75,18 @@ ifeq ($(run-built-tests),yes) $(objpfx)tst-svc-cmp.out: tst-svc.expect $(objpfx)tst-svc.out cmp $^ > $@; \ $(evaluate-test) + +LOCALES := de_DE.UTF-8 en_US.ISO-8859-1 en_US.UTF-8 \ + tr_TR.ISO-8859-9 tr_TR.UTF-8 cs_CZ.UTF-8 \ + da_DK.ISO-8859-1 + +include ../gen-locales.mk + +$(objpfx)test-strcasecmp.out: $(gen-locales) +$(objpfx)test-strncasecmp.out: $(gen-locales) +$(objpfx)tst-strxfrm.out: $(gen-locales) +$(objpfx)tst-strxfrm2.out: $(gen-locales) +# bug-strcoll2 needs cs_CZ.UTF-8 and da_DK.ISO-8859-1. +$(objpfx)bug-strcoll2.out: $(gen-locales) + endif Index: glibc-2.22/string/bug-strcoll2.c =================================================================== --- /dev/null +++ glibc-2.22/string/bug-strcoll2.c @@ -0,0 +1,92 @@ +/* Bug 18589: sort-test.sh fails at random. + Copyright (C) 2015 Free Software Foundation, Inc. + This file is part of the GNU C Library. + + The GNU C Library is free software; you can redistribute it and/or + modify it under the terms of the GNU Lesser General Public + License as published by the Free Software Foundation; either + version 2.1 of the License, or (at your option) any later version. + + The GNU C Library is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + Lesser General Public License for more details. + + You should have received a copy of the GNU Lesser General Public + License along with the GNU C Library; if not, see + . */ + +#include +#include +#include + +/* An incorrect strcoll optimization resulted in incorrect + results from strcoll for cs_CZ and da_DK. */ + +int +test_cs_CZ (void) +{ + const char t1[] = "config"; + const char t2[] = "choose"; + if (setlocale (LC_ALL, "cs_CZ.UTF-8") == NULL) + { + perror ("setlocale"); + return 1; + } + /* In Czech the digraph ch sorts after c, therefore we expect + config to sort before choose. */ + int a = strcoll (t1, t2); + int b = strcoll (t2, t1); + printf ("strcoll (\"%s\", \"%s\") = %d\n", t1, t2, a); + printf ("strcoll (\"%s\", \"%s\") = %d\n", t2, t1, b); + if (a < 0 && b > 0) + { + puts ("PASS: config < choose"); + return 0; + } + else + { + puts ("FAIL: Wrong sorting in cs_CZ.UTF-8."); + return 1; + } +} + +int +test_da_DK (void) +{ + const char t1[] = "AS"; + const char t2[] = "AA"; + if (setlocale (LC_ALL, "da_DK.ISO-8859-1") == NULL) + { + perror ("setlocale"); + return 1; + } + /* AA should be treated as the last letter of the Danish alphabet, + hence sorting after AS. */ + int a = strcoll (t1, t2); + int b = strcoll (t2, t1); + printf ("strcoll (\"%s\", \"%s\") = %d\n", t1, t2, a); + printf ("strcoll (\"%s\", \"%s\") = %d\n", t2, t1, b); + if (a < 0 && b > 0) + { + puts ("PASS: AS < AA"); + return 0; + } + else + { + puts ("FAIL: Wrong sorting in da_DK.ISO-8859-1"); + return 1; + } +} + +static int +do_test (void) +{ + int err = 0; + err |= test_cs_CZ (); + err |= test_da_DK (); + return err; +} + +#define TEST_FUNCTION do_test () +#include "../test-skeleton.c" Index: glibc-2.22/string/strcoll_l.c =================================================================== --- glibc-2.22.orig/string/strcoll_l.c +++ glibc-2.22/string/strcoll_l.c @@ -29,7 +29,6 @@ # define STRING_TYPE char # define USTRING_TYPE unsigned char # define STRCOLL __strcoll_l -# define STRDIFF __strdiff # define STRCMP strcmp # define WEIGHT_H "../locale/weight.h" # define SUFFIX MB @@ -42,20 +41,6 @@ #include "../locale/localeinfo.h" #include WEIGHT_H -#define MASK_UTF8_7BIT (1 << 7) -#define MASK_UTF8_START (3 << 6) - -size_t -STRDIFF (const STRING_TYPE *s, const STRING_TYPE *t) -{ - size_t n; - - for (n = 0; *s != '\0' && *s++ == *t++; ++n) - continue; - - return n; -} - /* Track status while looking for sequences in a string. */ typedef struct { @@ -269,29 +254,9 @@ STRCOLL (const STRING_TYPE *s1, const ST const USTRING_TYPE *extra; const int32_t *indirect; - /* In case there is no locale specific sort order (C / POSIX). */ if (nrules == 0) return STRCMP (s1, s2); - /* Fast forward to the position of the first difference. Needs to be - encoding aware as the byte-by-byte comparison can stop in the middle - of a char sequence for multibyte encodings like UTF-8. */ - uint_fast32_t encoding = - current->values[_NL_ITEM_INDEX (_NL_COLLATE_ENCODING_TYPE)].word; - if (encoding != __cet_other) - { - size_t diff = STRDIFF (s1, s2); - if (diff > 0) - { - if (encoding == __cet_utf8 && (*(s1 + diff) & MASK_UTF8_7BIT) != 0) - do - diff--; - while (diff > 0 && (*(s1 + diff) & MASK_UTF8_START) != MASK_UTF8_START); - s1 += diff; - s2 += diff; - } - } - /* Catch empty strings. */ if (__glibc_unlikely (*s1 == '\0') || __glibc_unlikely (*s2 == '\0')) return (*s1 != '\0') - (*s2 != '\0'); @@ -358,8 +323,7 @@ STRCOLL (const STRING_TYPE *s1, const ST byte-level comparison to ensure that we don't waste time going through multiple passes for totally equal strings before proceeding to subsequent passes. */ - if (pass == 0 && encoding == __cet_other && - STRCMP (s1, s2) == 0) + if (pass == 0 && STRCMP (s1, s2) == 0) return result; else break; Index: glibc-2.22/string/tst-strxfrm2.c =================================================================== --- glibc-2.22.orig/string/tst-strxfrm2.c +++ glibc-2.22/string/tst-strxfrm2.c @@ -5,6 +5,8 @@ static int do_test (void) { + static const char test_locale[] = "de_DE.UTF-8"; + int res = 0; char buf[20]; @@ -38,9 +40,9 @@ do_test (void) res = 1; } - if (setlocale (LC_ALL, "de_DE.UTF-8") == NULL) + if (setlocale (LC_ALL, test_locale) == NULL) { - puts ("setlocale failed"); + printf ("cannot set locale \"%s\"\n", test_locale); res = 1; } else Index: glibc-2.22/wcsmbs/wcscoll_l.c =================================================================== --- glibc-2.22.orig/wcsmbs/wcscoll_l.c +++ glibc-2.22/wcsmbs/wcscoll_l.c @@ -23,7 +23,6 @@ #define STRING_TYPE wchar_t #define USTRING_TYPE wint_t #define STRCOLL __wcscoll_l -#define STRDIFF __wcsdiff #define STRCMP __wcscmp #define WEIGHT_H "../locale/weightwc.h" #define SUFFIX WC