Skip to content

buffer: search utf16le at every byte offset #64917

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
inoway46 wants to merge 1 commit into
base: main
Choose a base branch
from
+104 −46
Open

buffer: search utf16le at every byte offset #64917

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
12 changes: 12 additions & 0 deletions doc/api/buffer.md
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2092,6 +2092,10 @@ console.log(buf.fill('zz', 'hex'));
<!-- YAML
added: v5.3.0
changes:
- version: REPLACEME
pr-url: https://github.com/nodejs/node/pull/64917
description: UTF-16LE and UCS2 searches now inspect every byte offset.
Buffer and Uint8Array values are compared in their entirety.
- version: v26.1.0
pr-url: https://github.com/nodejs/node/pull/62390
description: Added the `end` parameter.
Expand Down Expand Up @@ -2160,6 +2164,10 @@ console.log(buf.includes('this', 4));
<!-- YAML
added: v1.5.0
changes:
- version: REPLACEME
pr-url: https://github.com/nodejs/node/pull/64917
description: UTF-16LE and UCS2 searches now inspect every byte offset.
Buffer and Uint8Array values are compared in their entirety.
- version: v26.1.0
pr-url: https://github.com/nodejs/node/pull/62390
description: Added the `end` parameter.
Expand Down Expand Up @@ -2343,6 +2351,10 @@ for (const key of buf.keys()) {
<!-- YAML
added: v6.0.0
changes:
- version: REPLACEME
pr-url: https://github.com/nodejs/node/pull/64917
description: UTF-16LE and UCS2 searches now inspect every byte offset.
Buffer and Uint8Array values are compared in their entirety.
- version: v26.1.0
pr-url: https://github.com/nodejs/node/pull/62390
description: Added the `end` parameter.
Expand Down
62 changes: 21 additions & 41 deletions src/node_buffer.cc
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1045,17 +1045,14 @@ void IndexOfString(const FunctionCallbackInfo<Value>& args) {
int64_t end_i64 = args[5].As<Integer>()->Value();

const char* haystack = buffer.data();
// Round down to the nearest multiple of 2 in case of UCS2.
const size_t haystack_length = (enc == UCS2) ?
buffer.length() &~ 1 : buffer.length(); // NOLINT(whitespace/operators)
const size_t haystack_length = buffer.length();

size_t needle_length;
if (!StringBytes::Size(isolate, needle, enc).To(&needle_length)) return;

// search_end is the exclusive upper bound of the search range.
size_t search_end = static_cast<size_t>(std::min(
std::max(end_i64, int64_t{0}), static_cast<int64_t>(haystack_length)));
if (enc == UCS2) search_end &= ~static_cast<size_t>(1);

int64_t opt_offset = IndexOfOffset(haystack_length,
offset_i64,
Expand Down Expand Up @@ -1102,27 +1099,27 @@ void IndexOfString(const FunctionCallbackInfo<Value>& args) {
if constexpr (IsBigEndian()) {
StringBytes::InlineDecoder decoder;
if (decoder.Decode(env, needle, enc).IsNothing()) return;
const uint16_t* decoded_string =
reinterpret_cast<const uint16_t*>(decoder.out());
const uint8_t* decoded_string =
reinterpret_cast<const uint8_t*>(decoder.out());

if (decoded_string == nullptr)
return args.GetReturnValue().Set(-1);

result = nbytes::SearchString(reinterpret_cast<const uint16_t*>(haystack),
search_end / 2,
result = nbytes::SearchString(reinterpret_cast<const uint8_t*>(haystack),
search_end,
decoded_string,
decoder.size() / 2,
offset / 2,
decoder.size(),
offset,
is_forward);
} else {
result = nbytes::SearchString(reinterpret_cast<const uint16_t*>(haystack),
search_end / 2,
needle_value.out(),
needle_value.length(),
offset / 2,
is_forward);
result = nbytes::SearchString(
reinterpret_cast<const uint8_t*>(haystack),
search_end,
reinterpret_cast<const uint8_t*>(needle_value.out()),
needle_length,
offset,
is_forward);
}
result *= 2;
} else if (enc == UTF8) {
Utf8Value needle_value(isolate, needle);
if (*needle_value == nullptr)
Expand Down Expand Up @@ -1163,8 +1160,6 @@ void IndexOfBuffer(const FunctionCallbackInfo<Value>& args) {
CHECK(args[4]->IsBoolean());
CHECK(args[5]->IsNumber());

enum encoding enc = static_cast<enum encoding>(args[3].As<Int32>()->Value());

Environment* env = Environment::GetCurrent(args);
THROW_AND_RETURN_UNLESS_BUFFER(env, args[0]);
THROW_AND_RETURN_UNLESS_BUFFER(env, args[1]);
Expand All @@ -1182,7 +1177,6 @@ void IndexOfBuffer(const FunctionCallbackInfo<Value>& args) {
// search_end is the exclusive upper bound of the search range.
size_t search_end = static_cast<size_t>(std::min(
std::max(end_i64, int64_t{0}), static_cast<int64_t>(haystack_length)));
if (enc == UCS2) search_end &= ~static_cast<size_t>(1);

int64_t opt_offset = IndexOfOffset(haystack_length,
offset_i64,
Expand Down Expand Up @@ -1218,27 +1212,13 @@ void IndexOfBuffer(const FunctionCallbackInfo<Value>& args) {
return args.GetReturnValue().Set(-1);
}

size_t result = search_end;

if (enc == UCS2) {
if (search_end < 2 || needle_length < 2) {
return args.GetReturnValue().Set(-1);
}
result = nbytes::SearchString(reinterpret_cast<const uint16_t*>(haystack),
search_end / 2,
reinterpret_cast<const uint16_t*>(needle),
needle_length / 2,
offset / 2,
is_forward);
result *= 2;
} else {
result = nbytes::SearchString(reinterpret_cast<const uint8_t*>(haystack),
search_end,
reinterpret_cast<const uint8_t*>(needle),
needle_length,
offset,
is_forward);
}
size_t result =
nbytes::SearchString(reinterpret_cast<const uint8_t*>(haystack),
search_end,
reinterpret_cast<const uint8_t*>(needle),
needle_length,
offset,
is_forward);

args.GetReturnValue().Set(result >= search_end ? -1
: static_cast<int>(result));
Expand Down
13 changes: 12 additions & 1 deletion test/parallel/test-buffer-includes.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -74,7 +74,18 @@ assert(b.includes(Buffer.from('f'), 5));
assert(b.includes(Buffer.from('f'), -1));
assert(!b.includes(Buffer.from('f'), 6));

assert(!Buffer.from('ff').includes(Buffer.from('f'), 1, 'ucs2'));
assert(Buffer.from('ff').includes(Buffer.from('f'), 1, 'ucs2'));

{
const oddIndexBuffer = Buffer.from('00aaaa', 'hex');
const oddIndexNeedle = Buffer.from('\uaaaa', 'utf16le');
assert(oddIndexBuffer.includes('\uaaaa', 0, 'utf16le'));
assert(!oddIndexBuffer.includes('\uaaaa', 0, 2, 'utf16le'));
assert(oddIndexBuffer.includes('\uaaaa', 0, 3, 'utf16le'));
assert(oddIndexBuffer.includes(oddIndexNeedle, 0, 'utf16le'));
assert(oddIndexBuffer.includes(
new Uint8Array(oddIndexNeedle), 0, 'utf16le'));
}

// test hex encoding
assert.strictEqual(
Expand Down
63 changes: 59 additions & 4 deletions test/parallel/test-buffer-indexof.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -81,7 +81,7 @@ assert.strictEqual(b.indexOf(Buffer.from('f'), 5), 5);
assert.strictEqual(b.indexOf(Buffer.from('f'), -1), 5);
assert.strictEqual(b.indexOf(Buffer.from('f'), 6), -1);

assert.strictEqual(Buffer.from('ff').indexOf(Buffer.from('f'), 1, 'ucs2'), -1);
assert.strictEqual(Buffer.from('ff').indexOf(Buffer.from('f'), 1, 'ucs2'), 1);

// Test invalid and uppercase encoding
assert.strictEqual(b.indexOf('b', 'utf8'), 1);
Expand Down Expand Up @@ -192,7 +192,7 @@ assert.strictEqual(Buffer.from('aaaa0').indexOf('30', 'hex'), 4);
assert.strictEqual(Buffer.from('aaaa00a').indexOf('3030', 'hex'), 4);

{
// Test usc2 and utf16le encoding
// Test ucs2 and utf16le encodings.

@inoway46 inoway46 Aug 1, 2026

Copy link
Copy Markdown
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Fix typo (usc2 → ucs2) and wording.

['ucs2', 'utf16le'].forEach((encoding) => {
const twoByteString = Buffer.from(
'\u039a\u0391\u03a3\u03a3\u0395', encoding);
Expand Down Expand Up @@ -308,6 +308,51 @@ assert.strictEqual(Buffer.from('aaaa').indexOf('你好', 'ucs2'), -1);
// Haystack has odd length, but the needle is UCS2.
assert.strictEqual(Buffer.from('aaaaa').indexOf('b', 'ucs2'), -1);

{
// Search UTF-16LE values at byte offsets that are not aligned to the start
// of the Buffer.
const value = '\u6881\u6882\u6881';
const valueBuffer = Buffer.from(value, 'utf16le');
assert.strictEqual(valueBuffer.indexOf('\u6881', 1, 'utf16le'), 4);
assert.strictEqual(valueBuffer.slice(1).indexOf('\u6881', 'utf16le'), 3);
assert.strictEqual(valueBuffer.indexOf('\u6881', -5, 'utf16le'), 4);
assert.strictEqual(valueBuffer.indexOf('\u6881', 1, 4, 'utf16le'), -1);
assert.strictEqual(valueBuffer.indexOf('\u6881', 1, 6, 'utf16le'), 4);

const prefixed = Buffer.alloc(7);
prefixed.write(value, 1, 'utf16le');
assert.strictEqual(prefixed.indexOf(value, 1, 'utf16le'), 1);

// UTF-16LE searches compare bytes at every offset, even when a match spans
// two code units. The same needle also occurs on a code unit boundary at 4.
const crossUnitBuffer = Buffer.from([
0x41, 0x00, 0x01, 0x00, 0x00, 0x01,
]);
assert.strictEqual(crossUnitBuffer.indexOf('\u0100', 0, 'utf16le'), 1);

const oddIndexBuffer = Buffer.from('00aaaa', 'hex');
const oddIndexNeedle = Buffer.from('\uaaaa', 'utf16le');
assert.strictEqual(oddIndexBuffer.indexOf('\uaaaa', 0, 'utf16le'), 1);
assert.strictEqual(oddIndexBuffer.indexOf('\uaaaa', 0, 2, 'utf16le'), -1);
assert.strictEqual(oddIndexBuffer.indexOf('\uaaaa', 0, 3, 'utf16le'), 1);
assert.strictEqual(oddIndexBuffer.indexOf(
oddIndexNeedle, 0, 'utf16le'), 1);
assert.strictEqual(oddIndexBuffer.indexOf(
new Uint8Array(oddIndexNeedle), 0, 'utf16le'), 1);
assert.strictEqual(oddIndexBuffer.lastIndexOf(
'\uaaaa', 2, 3, 'utf16le'), 1);
assert.strictEqual(oddIndexBuffer.lastIndexOf(
oddIndexNeedle, 2, 3, 'utf16le'), 1);

// The encoding argument only applies to string needles. Buffer needles are
// compared byte-for-byte, so an odd-length needle includes its final byte.
const oddLengthNeedle = Buffer.from([0x61, 0x00, 0xff]);
assert.strictEqual(Buffer.from([0x61, 0x00, 0x62, 0x00])
.indexOf(oddLengthNeedle, 0, 'utf16le'), -1);
assert.strictEqual(Buffer.from([0x61, 0x00, 0xff, 0x00])
.indexOf(oddLengthNeedle, 0, 'utf16le'), 0);
}

{
// Find substrings in Utf8.
const lengths = [1, 3, 15]; // Single char, simple and complex.
Expand Down Expand Up @@ -349,12 +394,18 @@ assert.strictEqual(Buffer.from('aaaaa').indexOf('b', 'ucs2'), -1);

const patternBufferUcs2 =
allCharsBufferUcs2.slice(index, index + length);
const expectedBufferIndex =
allCharsBufferUcs2.indexOf(patternBufferUcs2);
assert.strictEqual(
index, allCharsBufferUcs2.indexOf(patternBufferUcs2, 0, 'ucs2'));
expectedBufferIndex,
allCharsBufferUcs2.indexOf(patternBufferUcs2, 0, 'ucs2'));

const patternStringUcs2 = patternBufferUcs2.toString('ucs2');
const expectedStringIndex = allCharsBufferUcs2.indexOf(
Buffer.from(patternStringUcs2, 'ucs2'));
assert.strictEqual(
index, allCharsBufferUcs2.indexOf(patternStringUcs2, 0, 'ucs2'));
expectedStringIndex,
allCharsBufferUcs2.indexOf(patternStringUcs2, 0, 'ucs2'));
}
}
}
Expand Down Expand Up @@ -722,7 +773,11 @@ assert.strictEqual(reallyLong.lastIndexOf(pattern), 0);
assert.strictEqual(buf.indexOf('', 0, 3), 0);
assert.strictEqual(buf.indexOf('', 5, 3), 3);
assert.strictEqual(buf.indexOf(Buffer.from(''), 5, 3), 3);
assert.strictEqual(buf.indexOf('', 5, 3, 'utf16le'), 3);
assert.strictEqual(buf.indexOf(Buffer.from(''), 5, 3, 'utf16le'), 3);
assert.strictEqual(buf.indexOf('', 0, 0), 0);
assert.strictEqual(buf.lastIndexOf('', 5, 3), 3);
assert.strictEqual(buf.lastIndexOf(Buffer.from(''), 5, 3), 3);
assert.strictEqual(buf.lastIndexOf('', 5, 3, 'utf16le'), 3);
assert.strictEqual(buf.lastIndexOf(Buffer.from(''), 5, 3, 'utf16le'), 3);
}
Loading