Skip to content

[RFC] Add a locale for grapheme case-insensitive functions #18792

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Draft
wants to merge 11 commits into
base: master
Choose a base branch
from
+161 −47
Draft

[RFC] Add a locale for grapheme case-insensitive functions #18792

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
11 commits
Select commit Hold shift + click to select a range
e3237bb
[RFC] Add a locale for grapheme case-insensitive functions
youkidearitai Jun 7, 2025
7973e72
Separate parameter of grapheme_strstr and grapheme_stristr
youkidearitai Jun 7, 2025
608d448
Add locale for grapheme_levenshtein function
youkidearitai Jun 16, 2025
ff994a5
Fix signatures
youkidearitai Jun 30, 2025
f72e59f
minor changes
youkidearitai Jun 30, 2025
abc304b
Remove locale for grapheme_levenshetin
youkidearitai Jul 5, 2025
10b43c2
Add a collator strength and locale
youkidearitai Jul 8, 2025
6656fab
Fix php_intl.stub.php
youkidearitai Jul 8, 2025
e452334
Fix zpp mismatch
youkidearitai Jul 8, 2025
3431fd7
Add test for strength from Japanese nabe
youkidearitai Jul 8, 2025
a993d00
Modify default value for grapheme_stri* functions
youkidearitai Jul 9, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
79 changes: 55 additions & 24 deletions ext/intl/grapheme/grapheme_string.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -81,19 +81,22 @@ PHP_FUNCTION(grapheme_strlen)
/* {{{ Find position of first occurrence of a string within another */
PHP_FUNCTION(grapheme_strpos)
{
char *haystack, *needle;
size_t haystack_len, needle_len;
char *haystack, *needle, *locale = "";
size_t haystack_len, needle_len, locale_len;
const char *found;
zend_long loffset = 0;
int32_t offset = 0;
zend_long strength = UCOL_DEFAULT_STRENGTH;
size_t noffset = 0;
zend_long ret_pos;

ZEND_PARSE_PARAMETERS_START(2, 3)
ZEND_PARSE_PARAMETERS_START(2, 5)
Z_PARAM_STRING(haystack, haystack_len)
Z_PARAM_STRING(needle, needle_len)
Z_PARAM_OPTIONAL
Z_PARAM_LONG(loffset)
Z_PARAM_STRING(locale, locale_len)
Z_PARAM_LONG(strength)
ZEND_PARSE_PARAMETERS_END();

if ( OUTSIDE_STRING(loffset, haystack_len) ) {
Expand Down Expand Up @@ -121,7 +124,7 @@ PHP_FUNCTION(grapheme_strpos)
}

/* do utf16 part of the strpos */
ret_pos = grapheme_strpos_utf16(haystack, haystack_len, needle, needle_len, offset, NULL, 0 /* fIgnoreCase */, 0 /* last */ );
ret_pos = grapheme_strpos_utf16(haystack, haystack_len, needle, needle_len, offset, NULL, 0 /* fIgnoreCase */, 0, locale, strength /* last */ );

if ( ret_pos >= 0 ) {
RETURN_LONG(ret_pos);
Expand All @@ -134,19 +137,22 @@ PHP_FUNCTION(grapheme_strpos)
/* {{{ Find position of first occurrence of a string within another, ignoring case differences */
PHP_FUNCTION(grapheme_stripos)
{
char *haystack, *needle;
size_t haystack_len, needle_len;
char *haystack, *needle, *locale = "";
size_t haystack_len, needle_len, locale_len = 0;
const char *found;
zend_long loffset = 0;
int32_t offset = 0;
zend_long strength = UCOL_SECONDARY;
zend_long ret_pos;
int is_ascii;

ZEND_PARSE_PARAMETERS_START(2, 3)
ZEND_PARSE_PARAMETERS_START(2, 5)
Z_PARAM_STRING(haystack, haystack_len)
Z_PARAM_STRING(needle, needle_len)
Z_PARAM_OPTIONAL
Z_PARAM_LONG(loffset)
Z_PARAM_STRING(locale, locale_len)
Z_PARAM_LONG(strength)
ZEND_PARSE_PARAMETERS_END();

if ( OUTSIDE_STRING(loffset, haystack_len) ) {
Expand Down Expand Up @@ -185,7 +191,7 @@ PHP_FUNCTION(grapheme_stripos)
}

/* do utf16 part of the strpos */
ret_pos = grapheme_strpos_utf16(haystack, haystack_len, needle, needle_len, offset, NULL, 1 /* fIgnoreCase */, 0 /*last */ );
ret_pos = grapheme_strpos_utf16(haystack, haystack_len, needle, needle_len, offset, NULL, 1 /* fIgnoreCase */, 0, locale, strength /*last */ );

if ( ret_pos >= 0 ) {
RETURN_LONG(ret_pos);
Expand All @@ -200,17 +206,21 @@ PHP_FUNCTION(grapheme_stripos)
PHP_FUNCTION(grapheme_strrpos)
{
char *haystack, *needle;
size_t haystack_len, needle_len;
char *locale = "";
size_t haystack_len, needle_len, locale_len;
zend_long loffset = 0;
int32_t offset = 0;
zend_long strength = UCOL_DEFAULT_STRENGTH;
zend_long ret_pos;
int is_ascii;

ZEND_PARSE_PARAMETERS_START(2, 3)
ZEND_PARSE_PARAMETERS_START(2, 5)
Z_PARAM_STRING(haystack, haystack_len)
Z_PARAM_STRING(needle, needle_len)
Z_PARAM_OPTIONAL
Z_PARAM_LONG(loffset)
Z_PARAM_STRING(locale, locale_len)
Z_PARAM_LONG(strength)
ZEND_PARSE_PARAMETERS_END();

if ( OUTSIDE_STRING(loffset, haystack_len) ) {
Expand Down Expand Up @@ -242,7 +252,7 @@ PHP_FUNCTION(grapheme_strrpos)
/* else we need to continue via utf16 */
}

ret_pos = grapheme_strpos_utf16(haystack, haystack_len, needle, needle_len, offset, NULL, 0 /* f_ignore_case */, 1/* last */);
ret_pos = grapheme_strpos_utf16(haystack, haystack_len, needle, needle_len, offset, NULL, 0 /* f_ignore_case */, 1, locale, strength /* last */);

if ( ret_pos >= 0 ) {
RETURN_LONG(ret_pos);
Expand All @@ -257,18 +267,21 @@ PHP_FUNCTION(grapheme_strrpos)
/* {{{ Find position of last occurrence of a string within another, ignoring case */
PHP_FUNCTION(grapheme_strripos)
{
char *haystack, *needle;
size_t haystack_len, needle_len;
char *haystack, *needle, *locale = "";
size_t haystack_len, needle_len, locale_len = 0;
zend_long loffset = 0;
int32_t offset = 0;
zend_long strength = UCOL_SECONDARY;
zend_long ret_pos;
int is_ascii;

ZEND_PARSE_PARAMETERS_START(2, 3)
ZEND_PARSE_PARAMETERS_START(2, 5)
Z_PARAM_STRING(haystack, haystack_len)
Z_PARAM_STRING(needle, needle_len)
Z_PARAM_OPTIONAL
Z_PARAM_LONG(loffset)
Z_PARAM_STRING(locale, locale_len)
Z_PARAM_LONG(strength)
ZEND_PARSE_PARAMETERS_END();

if ( OUTSIDE_STRING(loffset, haystack_len) ) {
Expand Down Expand Up @@ -309,7 +322,7 @@ PHP_FUNCTION(grapheme_strripos)
/* else we need to continue via utf16 */
}

ret_pos = grapheme_strpos_utf16(haystack, haystack_len, needle, needle_len, offset, NULL, 1 /* f_ignore_case */, 1 /*last */);
ret_pos = grapheme_strpos_utf16(haystack, haystack_len, needle, needle_len, offset, NULL, 1 /* f_ignore_case */, 1, locale, strength /*last */);

if ( ret_pos >= 0 ) {
RETURN_LONG(ret_pos);
Expand All @@ -324,13 +337,14 @@ PHP_FUNCTION(grapheme_strripos)
/* {{{ Returns part of a string */
PHP_FUNCTION(grapheme_substr)
{
char *str;
char *str, *locale = "";
zend_string *u8_sub_str;
UChar *ustr;
size_t str_len;
size_t str_len, locale_len;
int32_t ustr_len;
zend_long lstart = 0, length = 0;
int32_t start = 0;
zend_long strength = UCOL_DEFAULT;
int iter_val;
UErrorCode status;
unsigned char u_break_iterator_buffer[U_BRK_SAFECLONE_BUFFERSIZE];
Expand All @@ -339,11 +353,13 @@ PHP_FUNCTION(grapheme_substr)
int32_t (*iter_func)(UBreakIterator *);
bool no_length = true;

ZEND_PARSE_PARAMETERS_START(2, 3)
ZEND_PARSE_PARAMETERS_START(2, 5)
Z_PARAM_STRING(str, str_len)
Z_PARAM_LONG(lstart)
Z_PARAM_OPTIONAL
Z_PARAM_LONG_OR_NULL(length, no_length)
Z_PARAM_STRING(locale, locale_len)
Z_PARAM_LONG(strength)
ZEND_PARSE_PARAMETERS_END();

if (lstart < INT32_MIN || lstart > INT32_MAX) {
Expand Down Expand Up @@ -537,17 +553,26 @@ PHP_FUNCTION(grapheme_substr)
/* {{{ strstr_common_handler */
static void strstr_common_handler(INTERNAL_FUNCTION_PARAMETERS, int f_ignore_case)
{
char *haystack, *needle;
char *haystack, *needle, *locale = "";
const char *found;
size_t haystack_len, needle_len;
size_t haystack_len, needle_len, locale_len = 0;
int32_t ret_pos, uchar_pos;
zend_long strength;
bool part = false;

ZEND_PARSE_PARAMETERS_START(2, 3)
if (f_ignore_case) {
strength = UCOL_SECONDARY;
} else {
strength = UCOL_DEFAULT_STRENGTH;
}

ZEND_PARSE_PARAMETERS_START(2, 5)
Z_PARAM_STRING(haystack, haystack_len)
Z_PARAM_STRING(needle, needle_len)
Z_PARAM_OPTIONAL
Z_PARAM_BOOL(part)
Z_PARAM_STRING(locale, locale_len)
Z_PARAM_LONG(strength)
ZEND_PARSE_PARAMETERS_END();

if ( !f_ignore_case ) {
Expand All @@ -574,7 +599,7 @@ static void strstr_common_handler(INTERNAL_FUNCTION_PARAMETERS, int f_ignore_cas
}

/* need to work in utf16 */
ret_pos = grapheme_strpos_utf16(haystack, haystack_len, needle, needle_len, 0, &uchar_pos, f_ignore_case, 0 /*last */ );
ret_pos = grapheme_strpos_utf16(haystack, haystack_len, needle, needle_len, 0, &uchar_pos, f_ignore_case, 0, locale, strength /*last */ );

if ( ret_pos < 0 ) {
RETURN_FALSE;
Expand Down Expand Up @@ -919,14 +944,19 @@ PHP_FUNCTION(grapheme_levenshtein)
zend_long cost_ins = 1;
zend_long cost_rep = 1;
zend_long cost_del = 1;
char *locale = "";
size_t locale_len = 0;
zend_long strength = UCOL_DEFAULT_STRENGTH;

ZEND_PARSE_PARAMETERS_START(2, 5)
ZEND_PARSE_PARAMETERS_START(2, 7)
Z_PARAM_STR(string1)
Z_PARAM_STR(string2)
Z_PARAM_OPTIONAL
Z_PARAM_LONG(cost_ins)
Z_PARAM_LONG(cost_rep)
Z_PARAM_LONG(cost_del)
Z_PARAM_STRING(locale, locale_len)
Z_PARAM_LONG(strength)
ZEND_PARSE_PARAMETERS_END();

if (cost_ins <= 0 || cost_ins > UINT_MAX / 4) {
Expand Down Expand Up @@ -1043,14 +1073,15 @@ PHP_FUNCTION(grapheme_levenshtein)
RETVAL_FALSE;
goto out_bi2;
}
UCollator *collator = ucol_open("", &ustatus);
UCollator *collator = ucol_open(locale, &ustatus);
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

If the locale parameter is passed in as an actual NULL, wouldn't this fail?

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Yes, that's right. I should change to signature of RFC. Thanks.

if (U_FAILURE(ustatus)) {
intl_error_set_code(NULL, ustatus);

intl_error_set_custom_msg(NULL, "Error on ucol_open", 0);
RETVAL_FALSE;
goto out_collator;
}
ucol_setStrength(collator, strength);

zend_long *p1, *p2, *tmp;
p1 = safe_emalloc((size_t) strlen_2 + 1, sizeof(zend_long), 0);
Expand Down
16 changes: 7 additions & 9 deletions ext/intl/grapheme/grapheme_util.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -94,7 +94,7 @@ void grapheme_substr_ascii(char *str, size_t str_len, int32_t f, int32_t l, char


/* {{{ grapheme_strpos_utf16 - strrpos using utf16*/
int32_t grapheme_strpos_utf16(char *haystack, size_t haystack_len, char *needle, size_t needle_len, int32_t offset, int32_t *puchar_pos, int f_ignore_case, int last)
int32_t grapheme_strpos_utf16(char *haystack, size_t haystack_len, char *needle, size_t needle_len, int32_t offset, int32_t *puchar_pos, int f_ignore_case, int last, char* locale, int32_t strength)
{
UChar *uhaystack = NULL, *uneedle = NULL;
int32_t uhaystack_len = 0, uneedle_len = 0, char_pos, ret_pos, offset_pos = 0;
Expand Down Expand Up @@ -136,16 +136,14 @@ int32_t grapheme_strpos_utf16(char *haystack, size_t haystack_len, char *needle,
}

status = U_ZERO_ERROR;
src = usearch_open(uneedle, uneedle_len, uhaystack, uhaystack_len, "", bi, &status);
src = usearch_open(uneedle, uneedle_len, uhaystack, uhaystack_len, locale, bi, &status);
STRPOS_CHECK_STATUS(status, "Error creating search object");

if(f_ignore_case) {
UCollator *coll = usearch_getCollator(src);
status = U_ZERO_ERROR;
ucol_setAttribute(coll, UCOL_STRENGTH, UCOL_SECONDARY, &status);
STRPOS_CHECK_STATUS(status, "Error setting collation strength");
usearch_reset(src);
}
UCollator *coll = usearch_getCollator(src);
ucol_setAttribute(coll, UCOL_STRENGTH, strength, &status);
STRPOS_CHECK_STATUS(status, "Error setting collation strength");
status = U_ZERO_ERROR;
usearch_reset(src);

if(offset != 0) {
offset_pos = grapheme_get_haystack_offset(bi, offset);
Expand Down
4 changes: 2 additions & 2 deletions ext/intl/grapheme/grapheme_util.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -25,8 +25,8 @@ zend_long grapheme_ascii_check(const unsigned char *day, size_t len);
void grapheme_substr_ascii(char *str, size_t str_len, int32_t f, int32_t l, char **sub_str, int32_t *sub_str_len);
zend_long grapheme_strrpos_ascii(char *haystack, size_t haystack_len, char *needle, size_t needle_len, int32_t offset);

int32_t grapheme_strrpos_utf16(char *haystack, size_t haystack_len, char *needle, size_t needle_len, int32_t offset, int f_ignore_case);
int32_t grapheme_strpos_utf16(char *haystack, size_t haystack_len, char *needle, size_t needle_len, int32_t offset, int *puchar_pos, int f_ignore_case, int last);
int32_t grapheme_strrpos_utf16(char *haystack, size_t haystack_len, char *needle, size_t needle_len, int32_t offset, int f_ignore_case, char* locale, int32_t strength);
int32_t grapheme_strpos_utf16(char *haystack, size_t haystack_len, char *needle, size_t needle_len, int32_t offset, int *puchar_pos, int f_ignore_case, int last, char* locale, int32_t strength);

int32_t grapheme_split_string(const UChar *text, int32_t text_length, int boundary_array[], int boundary_array_len );

Expand Down
26 changes: 18 additions & 8 deletions ext/intl/php_intl.stub.php
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -166,6 +166,16 @@
* @cvalue UIDNA_ERROR_CONTEXTJ
*/
const IDNA_ERROR_CONTEXTJ = UNKNOWN;
/**
* @var int
* @cvalue UCOL_DEFAULT_STRENGTH
*/
const UCOL_DEFAULT_STRENGTH = UNKNOWN;
/**
* @var int
* @cvalue UCOL_SECONDARY
*/
const UCOL_SECONDARY = UNKNOWN;

class IntlException extends Exception
{
Expand Down Expand Up @@ -431,23 +441,23 @@ function numfmt_get_error_message(NumberFormatter $formatter): string {}

function grapheme_strlen(string $string): int|false|null {}

function grapheme_strpos(string $haystack, string $needle, int $offset = 0): int|false {}
function grapheme_strpos(string $haystack, string $needle, int $offset = 0, string $locale = "", int $strength = UCOL_DEFAULT_STRENGTH): int|false {}

function grapheme_stripos(string $haystack, string $needle, int $offset = 0): int|false {}
function grapheme_stripos(string $haystack, string $needle, int $offset = 0, string $locale = "", int $strength = UCOL_SECONDARY): int|false {}

function grapheme_strrpos(string $haystack, string $needle, int $offset = 0): int|false {}
function grapheme_strrpos(string $haystack, string $needle, int $offset = 0, string $locale = "", int $strength = UCOL_DEFAULT_STRENGTH): int|false {}

function grapheme_strripos(string $haystack, string $needle, int $offset = 0): int|false {}
function grapheme_strripos(string $haystack, string $needle, int $offset = 0, string $locale = "", int $strength = UCOL_SECONDARY): int|false {}

function grapheme_substr(string $string, int $offset, ?int $length = null): string|false {}
function grapheme_substr(string $string, int $offset, ?int $length = null, string $locale = "", int $strength = UCOL_DEFAULT_STRENGTH): string|false {}

function grapheme_strstr(string $haystack, string $needle, bool $beforeNeedle = false): string|false {}
function grapheme_strstr(string $haystack, string $needle, bool $beforeNeedle = false, string $locale = "", int $strength = UCOL_DEFAULT_STRENGTH): string|false {}

function grapheme_stristr(string $haystack, string $needle, bool $beforeNeedle = false): string|false {}
function grapheme_stristr(string $haystack, string $needle, bool $beforeNeedle = false, string $locale = "", int $strength = UCOL_SECONDARY): string|false {}

function grapheme_str_split(string $string, int $length = 1): array|false {}

function grapheme_levenshtein(string $string1, string $string2, int $insertion_cost = 1, int $replacement_cost = 1, int $deletion_cost = 1): int|false {}
function grapheme_levenshtein(string $string1, string $string2, int $insertion_cost = 1, int $replacement_cost = 1, int $deletion_cost = 1, string $locale = "", int $strength = UCOL_DEFAULT_STRENGTH): int|false {}

/** @param int $next */
function grapheme_extract(string $haystack, int $size, int $type = GRAPHEME_EXTR_COUNT, int $offset = 0, &$next = null): string|false {}
Expand Down
Loading