Skip to content

Commit

Permalink
handle non-ascii whitespace
Browse files Browse the repository at this point in the history
  • Loading branch information
HertzDevil committed Nov 15, 2024
1 parent d8baa1b commit 83276b0
Show file tree
Hide file tree
Showing 4 changed files with 22 additions and 30 deletions.
3 changes: 3 additions & 0 deletions spec/std/string_spec.cr
Original file line number Diff line number Diff line change
Expand Up @@ -504,6 +504,7 @@ describe "String" do
" 1234.56 ".to_f?(whitespace: false).should be_nil
expect_raises(ArgumentError) { " 1234.56foo".to_f }
" 1234.56foo".to_f?.should be_nil
"\u{A0}\u{2028}\u{2029}1234.56\u{A0}\u{2028}\u{2029}".to_f.should eq(1234.56_f64)
"123.45 x".to_f64(strict: false).should eq(123.45_f64)
expect_raises(ArgumentError) { "x1.2".to_f64 }
"x1.2".to_f64?.should be_nil
Expand Down Expand Up @@ -548,6 +549,7 @@ describe "String" do
" 1234.56 ".to_f32?(whitespace: false).should be_nil
expect_raises(ArgumentError) { " 1234.56foo".to_f32 }
" 1234.56foo".to_f32?.should be_nil
"\u{A0}\u{2028}\u{2029}1234.56\u{A0}\u{2028}\u{2029}".to_f32.should eq(1234.56_f32)
"123.45 x".to_f32(strict: false).should eq(123.45_f32)
expect_raises(ArgumentError) { "x1.2".to_f32 }
"x1.2".to_f32?.should be_nil
Expand Down Expand Up @@ -591,6 +593,7 @@ describe "String" do
" 1234.56 ".to_f64?(whitespace: false).should be_nil
expect_raises(ArgumentError) { " 1234.56foo".to_f64 }
" 1234.56foo".to_f64?.should be_nil
"\u{A0}\u{2028}\u{2029}1234.56\u{A0}\u{2028}\u{2029}".to_f64.should eq(1234.56_f64)
"123.45 x".to_f64(strict: false).should eq(123.45_f64)
expect_raises(ArgumentError) { "x1.2".to_f64 }
"x1.2".to_f64?.should be_nil
Expand Down
35 changes: 14 additions & 21 deletions src/float/fast_float.cr
Original file line number Diff line number Diff line change
Expand Up @@ -42,11 +42,14 @@ struct Float
finish = start + str.bytesize
options = ParseOptionsT(typeof(str.to_unsafe.value)).new(format: :general)

ret = BinaryFormat_Float64.new.from_chars_advanced(start, finish, pointerof(value), options, whitespace: whitespace)
if ret.ec == Errno::NONE
if trailing_chars_allowed?(ret.ptr, finish, whitespace, strict)
value
end
if whitespace
start += str.calc_excess_left
finish -= str.calc_excess_right
end

ret = BinaryFormat_Float64.new.from_chars_advanced(start, finish, pointerof(value), options)
if ret.ec == Errno::NONE && (!strict || ret.ptr == finish)
value
end
end

Expand All @@ -56,24 +59,14 @@ struct Float
finish = start + str.bytesize
options = ParseOptionsT(typeof(str.to_unsafe.value)).new(format: :general)

ret = BinaryFormat_Float32.new.from_chars_advanced(start, finish, pointerof(value), options, whitespace: whitespace)
if ret.ec == Errno::NONE
if trailing_chars_allowed?(ret.ptr, finish, whitespace, strict)
value
end
if whitespace
start += str.calc_excess_left
finish -= str.calc_excess_right
end
end

private def self.trailing_chars_allowed?(ptr, finish, whitespace, strict)
if strict
if whitespace
while ptr < finish && ptr.value.unsafe_chr.ascii_whitespace?
ptr += 1
end
end
ptr == finish
else
true
ret = BinaryFormat_Float32.new.from_chars_advanced(start, finish, pointerof(value), options)
if ret.ec == Errno::NONE && (!strict || ret.ptr == finish)
value
end
end
end
Expand Down
8 changes: 1 addition & 7 deletions src/float/fast_float/parse_number.cr
Original file line number Diff line number Diff line change
Expand Up @@ -172,18 +172,12 @@ module Float::FastFloat
FromCharsResultT(UC).new(ptr, ec)
end

# NOTE(crystal): *whitespace* is a normal argument
def from_chars_advanced(first : UC*, last : UC*, value : T*, options : ParseOptionsT(UC), whitespace : Bool) : FromCharsResultT(UC) forall UC
def from_chars_advanced(first : UC*, last : UC*, value : T*, options : ParseOptionsT(UC)) : FromCharsResultT(UC) forall UC
{% raise "only some floating-point types are supported" unless T == Float32 || T == Float64 %}

# TODO(crystal): support UInt16 and UInt32
{% raise "only UInt8 is supported" unless UC == UInt8 %}

if whitespace
while first != last && first.value.unsafe_chr.ascii_whitespace?
first += 1
end
end
if first == last
return FromCharsResultT(UC).new(first, Errno::EINVAL)
end
Expand Down
6 changes: 4 additions & 2 deletions src/string.cr
Original file line number Diff line number Diff line change
Expand Up @@ -2105,7 +2105,8 @@ class String
remove_excess_left(excess_left)
end

private def calc_excess_right
# :nodoc:
def calc_excess_right
if single_byte_optimizable?
i = bytesize - 1
while i >= 0 && to_unsafe[i].unsafe_chr.ascii_whitespace?
Expand Down Expand Up @@ -2143,7 +2144,8 @@ class String
bytesize - byte_index
end

private def calc_excess_left
# :nodoc:
def calc_excess_left
if single_byte_optimizable?
excess_left = 0
# All strings end with '\0', and it's not a whitespace
Expand Down

0 comments on commit 83276b0

Please sign in to comment.