Fix #72809: Locale::lookup() wrong result with canonicalize option

Canonicalization converts the locale to ICU format[1].  However, the
lookup described in RFC 4647, section 3.4, is about POSIX format.  To
make that lookup work for ICU format, we also need to cater to keyword
separators.

The results are somewhat unexpected, but apparently canonical lookup is
explicitly supposed to return canonical language tags[2].

[1] <https://unicode-org.github.io/icu/userguide/locale/#canonicalization>
[2] <https://github.com/php/php-src/blob/php-7.4.20/ext/intl/locale/locale_methods.c#L1504>

Closes phpGH-7151.

Author: cmb69

NEWS

@@ -2,6 +2,9 @@ PHP                                                                        NEWS
 |||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||
 ?? ??? ????, PHP 7.4.22
 
+- Intl:
+  . Fixed bug #72809 (Locale::lookup() wrong result with canonicalize option).
+    (cmb)
 
 01 Jul 2021, PHP 7.4.21
 

ext/intl/locale/locale_methods.c

@@ -149,7 +149,7 @@ static zend_off_t getStrrtokenPos(char* str, zend_off_t savedPos)
 	zend_off_t i;
 
 	for(i=savedPos-1; i>=0; i--) {
-		if(isIDSeparator(*(str+i)) ){
+		if(isIDSeparator(*(str+i)) || isKeywordSeparator(*(str+i))){
 			/* delimiter found; check for singleton */
 			if(i>=2 && isIDSeparator(*(str+i-2)) ){
 				/* a singleton; so send the position of token before the singleton */

ext/intl/tests/locale/bug72809.phpt

@@ -0,0 +1,18 @@
+--TEST--
+Bug #72809 (Locale::lookup() wrong result with canonicalize option)
+--SKIPIF--
+<?php
+if (!extension_loaded('intl')) die("skip intl extension not avaible");
+?>
+--FILE--
+<?php
+var_dump(
+    Locale::lookup(['en', 'en-US'], 'en-US-u-cu-EUR-tz-deber-fw-mon', true),
+    Locale::lookup(['en', 'en_US'], 'en_US@currency=eur;fw=mon;timezone=Europe/Berlin', false),
+    Locale::lookup(['en', 'en_US'], 'en_US@currency=eur;fw=mon;timezone=Europe/Berlin', true),
+);
+?>
+--EXPECT--
+string(5) "en_us"
+string(5) "en_US"
+string(5) "en_us"