StatProfilerHTML.jl report
Generated on tor 10 okt 2019 11:38:33
File source code
Line Exclusive Inclusive Code
1 # This file is a part of Julia. License is MIT: https://julialang.org/license
2
3 ## work with AbstractVector{UInt8} via I/O primitives ##
4
5 # Stateful string
6 mutable struct GenericIOBuffer{T<:AbstractVector{UInt8}} <: IO
7 data::T # T should support: getindex, setindex!, length, copyto!, and resize!
8 readable::Bool
9 writable::Bool
10 seekable::Bool # if not seekable, implementation is free to destroy (compact) past read data
11 append::Bool # add data at end instead of at pointer
12 size::Int # end pointer (and write pointer if append == true)
13 maxsize::Int # fixed array size (typically pre-allocated)
14 ptr::Int # read (and maybe write) pointer
15 mark::Int # reset mark location for ptr (or <0 for no mark)
16
17 function GenericIOBuffer{T}(data::T, readable::Bool, writable::Bool, seekable::Bool, append::Bool,
18 maxsize::Integer) where T<:AbstractVector{UInt8}
19 @assert !has_offset_axes(data)
20 new(data,readable,writable,seekable,append,length(data),maxsize,1,-1)
21 end
22 end
23 const IOBuffer = GenericIOBuffer{Vector{UInt8}}
24
25 function GenericIOBuffer(data::T, readable::Bool, writable::Bool, seekable::Bool, append::Bool,
26 maxsize::Integer) where T<:AbstractVector{UInt8}
27 GenericIOBuffer{T}(data, readable, writable, seekable, append, maxsize)
28 end
29
30 # allocate Vector{UInt8}s for IOBuffer storage that can efficiently become Strings
31 5 (11.90%)
5 (11.90%) samples spent in StringVector
0 (ex.), 5 (100.00%) (incl.) when called from #IOBuffer#302 line 114
5 (100.00%) samples spent calling _string_n
StringVector(n::Integer) = unsafe_wrap(Vector{UInt8}, _string_n(n))
32
33 # IOBuffers behave like Files. They are typically readable and writable. They are seekable. (They can be appendable).
34
35 """
36 IOBuffer([data::AbstractVector{UInt8}]; keywords...) -> IOBuffer
37
38 Create an in-memory I/O stream, which may optionally operate on a pre-existing array.
39
40 It may take optional keyword arguments:
41 - `read`, `write`, `append`: restricts operations to the buffer; see `open` for details.
42 - `truncate`: truncates the buffer size to zero length.
43 - `maxsize`: specifies a size beyond which the buffer may not be grown.
44 - `sizehint`: suggests a capacity of the buffer (`data` must implement `sizehint!(data, size)`).
45
46 When `data` is not given, the buffer will be both readable and writable by default.
47
48 # Examples
49 ```jldoctest
50 julia> io = IOBuffer();
51
52 julia> write(io, "JuliaLang is a GitHub organization.", " It has many members.")
53 56
54
55 julia> String(take!(io))
56 "JuliaLang is a GitHub organization. It has many members."
57
58 julia> io = IOBuffer(b"JuliaLang is a GitHub organization.")
59 IOBuffer(data=UInt8[...], readable=true, writable=false, seekable=true, append=false, size=35, maxsize=Inf, ptr=1, mark=-1)
60
61 julia> read(io, String)
62 "JuliaLang is a GitHub organization."
63
64 julia> write(io, "This isn't writable.")
65 ERROR: ArgumentError: ensureroom failed, IOBuffer is not writeable
66
67 julia> io = IOBuffer(UInt8[], read=true, write=true, maxsize=34)
68 IOBuffer(data=UInt8[...], readable=true, writable=true, seekable=true, append=false, size=0, maxsize=34, ptr=1, mark=-1)
69
70 julia> write(io, "JuliaLang is a GitHub organization.")
71 34
72
73 julia> String(take!(io))
74 "JuliaLang is a GitHub organization"
75
76 julia> length(read(IOBuffer(b"data", read=true, truncate=false)))
77 4
78
79 julia> length(read(IOBuffer(b"data", read=true, truncate=true)))
80 0
81 ```
82 """
83 function IOBuffer(
84 data::AbstractVector{UInt8};
85 read::Union{Bool,Nothing}=nothing,
86 write::Union{Bool,Nothing}=nothing,
87 append::Union{Bool,Nothing}=nothing,
88 truncate::Union{Bool,Nothing}=nothing,
89 maxsize::Integer=typemax(Int),
90 sizehint::Union{Integer,Nothing}=nothing)
91 if maxsize < 0
92 throw(ArgumentError("negative maxsize"))
93 end
94 if sizehint !== nothing
95 sizehint!(data, sizehint)
96 end
97 flags = open_flags(read=read, write=write, append=append, truncate=truncate)
98 buf = GenericIOBuffer(data, flags.read, flags.write, true, flags.append, Int(maxsize))
99 if flags.truncate
100 buf.size = 0
101 end
102 return buf
103 end
104
105 function IOBuffer(;
106 read::Union{Bool,Nothing}=true,
107 write::Union{Bool,Nothing}=true,
108 append::Union{Bool,Nothing}=nothing,
109 truncate::Union{Bool,Nothing}=true,
110 maxsize::Integer=typemax(Int),
111
12 (28.57%) samples spent in #IOBuffer#302
1 (100.00%) (ex.), 12 (100.00%) (incl.) when called from Type line 0
sizehint::Union{Integer,Nothing}=nothing)
112 size = sizehint !== nothing ? Int(sizehint) : maxsize != typemax(Int) ? Int(maxsize) : 32
113 flags = open_flags(read=read, write=write, append=append, truncate=truncate)
114 11 (26.19%)
6 (54.55%) samples spent calling StringVector
5 (45.45%) samples spent calling StringVector
buf = IOBuffer(
115 StringVector(size),
116 read=flags.read,
117 write=flags.write,
118 append=flags.append,
119 truncate=flags.truncate,
120 maxsize=maxsize)
121 fill!(buf.data, 0)
122 1 (2.38%) 1 (2.38%) return buf
123 end
124
125 # PipeBuffers behave like Unix Pipes. They are typically readable and writable, they act appendable, and are not seekable.
126
127 """
128 PipeBuffer(data::Vector{UInt8}=UInt8[]; maxsize::Integer = typemax(Int))
129
130 An [`IOBuffer`](@ref) that allows reading and performs writes by appending.
131 Seeking and truncating are not supported.
132 See [`IOBuffer`](@ref) for the available constructors.
133 If `data` is given, creates a `PipeBuffer` to operate on a data vector,
134 optionally specifying a size beyond which the underlying `Array` may not be grown.
135 """
136 PipeBuffer(data::Vector{UInt8}=UInt8[]; maxsize::Int = typemax(Int)) =
137 GenericIOBuffer(data,true,true,false,true,maxsize)
138 PipeBuffer(maxsize::Integer) = (x = PipeBuffer(StringVector(maxsize), maxsize = maxsize); x.size=0; x)
139
140 function copy(b::GenericIOBuffer)
141 ret = typeof(b)(b.writable ? copy(b.data) : b.data,
142 b.readable, b.writable, b.seekable, b.append, b.maxsize)
143 ret.size = b.size
144 ret.ptr = b.ptr
145 return ret
146 end
147
148 show(io::IO, b::GenericIOBuffer) = print(io, "IOBuffer(data=UInt8[...], ",
149 "readable=", b.readable, ", ",
150 "writable=", b.writable, ", ",
151 "seekable=", b.seekable, ", ",
152 "append=", b.append, ", ",
153 "size=", b.size, ", ",
154 "maxsize=", b.maxsize == typemax(Int) ? "Inf" : b.maxsize, ", ",
155 "ptr=", b.ptr, ", ",
156 "mark=", b.mark, ")")
157
158 function unsafe_read(from::GenericIOBuffer, p::Ptr{UInt8}, nb::UInt)
159 from.readable || throw(ArgumentError("read failed, IOBuffer is not readable"))
160 avail = bytesavailable(from)
161 adv = min(avail, nb)
162 GC.@preserve from unsafe_copyto!(p, pointer(from.data, from.ptr), adv)
163 from.ptr += adv
164 if nb > avail
165 throw(EOFError())
166 end
167 nothing
168 end
169
170 function read(from::GenericIOBuffer, T::Union{Type{Int16},Type{UInt16},Type{Int32},Type{UInt32},Type{Int64},Type{UInt64},Type{Int128},Type{UInt128},Type{Float16},Type{Float32},Type{Float64}})
171 from.readable || throw(ArgumentError("read failed, IOBuffer is not readable"))
172 avail = bytesavailable(from)
173 nb = sizeof(T)
174 if nb > avail
175 throw(EOFError())
176 end
177 GC.@preserve from begin
178 ptr::Ptr{T} = pointer(from.data, from.ptr)
179 x = unsafe_load(ptr)
180 end
181 from.ptr += nb
182 return x
183 end
184
185 function read_sub(from::GenericIOBuffer, a::AbstractArray{T}, offs, nel) where T
186 @assert !has_offset_axes(a)
187 from.readable || throw(ArgumentError("read failed, IOBuffer is not readable"))
188 if offs+nel-1 > length(a) || offs < 1 || nel < 0
189 throw(BoundsError())
190 end
191 if isbitstype(T) && isa(a,Array)
192 nb = UInt(nel * sizeof(T))
193 GC.@preserve a unsafe_read(from, pointer(a, offs), nb)
194 else
195 for i = offs:offs+nel-1
196 a[i] = read(to, T)
197 end
198 end
199 return a
200 end
201
202 @inline function read(from::GenericIOBuffer, ::Type{UInt8})
203 from.readable || throw(ArgumentError("read failed, IOBuffer is not readable"))
204 ptr = from.ptr
205 size = from.size
206 if ptr > size
207 throw(EOFError())
208 end
209 @inbounds byte = from.data[ptr]
210 from.ptr = ptr + 1
211 return byte
212 end
213
214 function peek(from::GenericIOBuffer)
215 from.readable || throw(ArgumentError("read failed, IOBuffer is not readable"))
216 if from.ptr > from.size
217 throw(EOFError())
218 end
219 return from.data[from.ptr]
220 end
221
222 read(from::GenericIOBuffer, ::Type{Ptr{T}}) where {T} = convert(Ptr{T}, read(from, UInt))
223
224 isreadable(io::GenericIOBuffer) = io.readable
225 iswritable(io::GenericIOBuffer) = io.writable
226
227 # TODO: GenericIOBuffer is not iterable, so doesn't really have a length.
228 # This should maybe be sizeof() instead.
229 #length(io::GenericIOBuffer) = (io.seekable ? io.size : bytesavailable(io))
230 bytesavailable(io::GenericIOBuffer) = io.size - io.ptr + 1
231 position(io::GenericIOBuffer) = io.ptr-1
232
233 function skip(io::GenericIOBuffer, n::Integer)
234 seekto = io.ptr + n
235 n < 0 && return seek(io, seekto-1) # Does error checking
236 io.ptr = min(seekto, io.size+1)
237 return io
238 end
239
240 function seek(io::GenericIOBuffer, n::Integer)
241 if !io.seekable
242 ismarked(io) || throw(ArgumentError("seek failed, IOBuffer is not seekable and is not marked"))
243 n == io.mark || throw(ArgumentError("seek failed, IOBuffer is not seekable and n != mark"))
244 end
245 # TODO: REPL.jl relies on the fact that this does not throw (by seeking past the beginning or end
246 # of an GenericIOBuffer), so that would need to be fixed in order to throw an error here
247 #(n < 0 || n > io.size) && throw(ArgumentError("Attempted to seek outside IOBuffer boundaries."))
248 #io.ptr = n+1
249 io.ptr = max(min(n+1, io.size+1), 1)
250 return io
251 end
252
253 function seekend(io::GenericIOBuffer)
254 io.ptr = io.size+1
255 return io
256 end
257
258 function truncate(io::GenericIOBuffer, n::Integer)
259 io.writable || throw(ArgumentError("truncate failed, IOBuffer is not writeable"))
260 io.seekable || throw(ArgumentError("truncate failed, IOBuffer is not seekable"))
261 n < 0 && throw(ArgumentError("truncate failed, n bytes must be ≥ 0, got $n"))
262 n > io.maxsize && throw(ArgumentError("truncate failed, $(n) bytes is exceeds IOBuffer maxsize $(io.maxsize)"))
263 if n > length(io.data)
264 resize!(io.data, n)
265 end
266 io.data[io.size+1:n] .= 0
267 io.size = n
268 io.ptr = min(io.ptr, n+1)
269 ismarked(io) && io.mark > n && unmark(io)
270 return io
271 end
272
273 function compact(io::GenericIOBuffer)
274 io.writable || throw(ArgumentError("compact failed, IOBuffer is not writeable"))
275 io.seekable && throw(ArgumentError("compact failed, IOBuffer is seekable"))
276 local ptr::Int, bytes_to_move::Int
277 if ismarked(io) && io.mark < io.ptr
278 if io.mark == 0 return end
279 ptr = io.mark
280 bytes_to_move = bytesavailable(io) + (io.ptr-io.mark)
281 else
282 ptr = io.ptr
283 bytes_to_move = bytesavailable(io)
284 end
285 copyto!(io.data, 1, io.data, ptr, bytes_to_move)
286 io.size -= ptr - 1
287 io.ptr -= ptr - 1
288 io.mark -= ptr - 1
289 return io
290 end
291
292 @inline ensureroom(io::GenericIOBuffer, nshort::Int) = ensureroom(io, UInt(nshort))
293
1 (2.38%) samples spent in ensureroom
1 (100.00%) (ex.), 1 (100.00%) (incl.) when called from unsafe_write line 402
@inline function ensureroom(io::GenericIOBuffer, nshort::UInt)
294 1 (2.38%) 1 (2.38%) io.writable || throw(ArgumentError("ensureroom failed, IOBuffer is not writeable"))
295 if !io.seekable
296 nshort >= 0 || throw(ArgumentError("ensureroom failed, requested number of bytes must be ≥ 0, got $nshort"))
297 if !ismarked(io) && io.ptr > 1 && io.size <= io.ptr - 1
298 io.ptr = 1
299 io.size = 0
300 else
301 datastart = ismarked(io) ? io.mark : io.ptr
302 if (io.size+nshort > io.maxsize) ||
303 (datastart > 4096 && datastart > io.size - io.ptr) ||
304 (datastart > 262144)
305 # apply somewhat arbitrary heuristics to decide when to destroy
306 # old, read data to make more room for new data
307 compact(io)
308 end
309 end
310 end
311 n = min(nshort + (io.append ? io.size : io.ptr-1), io.maxsize)
312 if n > length(io.data)
313 resize!(io.data, n)
314 end
315 return io
316 end
317
318 eof(io::GenericIOBuffer) = (io.ptr-1 == io.size)
319
320 @noinline function close(io::GenericIOBuffer{T}) where T
321 io.readable = false
322 io.writable = false
323 io.seekable = false
324 io.size = 0
325 io.maxsize = 0
326 io.ptr = 1
327 io.mark = -1
328 if io.writable
329 resize!(io.data, 0)
330 end
331 nothing
332 end
333
334 isopen(io::GenericIOBuffer) = io.readable || io.writable || io.seekable || bytesavailable(io) > 0
335
336 """
337 take!(b::IOBuffer)
338
339 Obtain the contents of an `IOBuffer` as an array, without copying. Afterwards, the
340 `IOBuffer` is reset to its initial state.
341
342 # Examples
343 ```jldoctest
344 julia> io = IOBuffer();
345
346 julia> write(io, "JuliaLang is a GitHub organization.", " It has many members.")
347 56
348
349 julia> String(take!(io))
350 "JuliaLang is a GitHub organization. It has many members."
351 ```
352 """
353 function take!(io::GenericIOBuffer)
354 ismarked(io) && unmark(io)
355 if io.seekable
356 nbytes = io.size
357 data = copyto!(StringVector(nbytes), 1, io.data, 1, nbytes)
358 else
359 nbytes = bytesavailable(io)
360 data = read!(io,StringVector(nbytes))
361 end
362 if io.writable
363 io.ptr = 1
364 io.size = 0
365 end
366 return data
367 end
368 function take!(io::IOBuffer)
369 ismarked(io) && unmark(io)
370 if io.seekable
371 data = io.data
372 if io.writable
373 maxsize = (io.maxsize == typemax(Int) ? 0 : min(length(io.data),io.maxsize))
374 io.data = StringVector(maxsize)
375 else
376 data = copy(data)
377 end
378 resize!(data,io.size)
379 else
380 nbytes = bytesavailable(io)
381 a = StringVector(nbytes)
382 data = read!(io, a)
383 end
384 if io.writable
385 io.ptr = 1
386 io.size = 0
387 end
388 return data
389 end
390
391 function write(to::GenericIOBuffer, from::GenericIOBuffer)
392 if to === from
393 from.ptr = from.size + 1
394 return 0
395 end
396 written::Int = write_sub(to, from.data, from.ptr, bytesavailable(from))
397 from.ptr += written
398 return written
399 end
400
401
3 (7.14%) samples spent in unsafe_write
0 (ex.), 1 (33.33%) (incl.) when called from unsafe_write line 509
1 (100.00%) (ex.), 2 (66.67%) (incl.) when called from write line 87
function unsafe_write(to::GenericIOBuffer, p::Ptr{UInt8}, nb::UInt)
402 1 (2.38%)
1 (100.00%) samples spent calling ensureroom
ensureroom(to, nb)
403 1 (2.38%) 1 (2.38%) ptr = (to.append ? to.size+1 : to.ptr)
404 written = Int(min(nb, length(to.data) - ptr + 1))
405 towrite = written
406 d = to.data
407 1 (2.38%)
1 (100.00%) samples spent calling >
while towrite > 0
408 @inbounds d[ptr] = unsafe_load(p)
409 ptr += 1
410 p += 1
411 towrite -= 1
412 end
413 to.size = max(to.size, ptr - 1)
414 if !to.append
415 to.ptr += written
416 end
417 return written
418 end
419
420 function write_sub(to::GenericIOBuffer, a::AbstractArray{UInt8}, offs, nel)
421 @assert !has_offset_axes(a)
422 if offs+nel-1 > length(a) || offs < 1 || nel < 0
423 throw(BoundsError())
424 end
425 GC.@preserve a unsafe_write(to, pointer(a, offs), UInt(nel))
426 end
427
428 @inline function write(to::GenericIOBuffer, a::UInt8)
429 ensureroom(to, UInt(1))
430 ptr = (to.append ? to.size+1 : to.ptr)
431 if ptr > to.maxsize
432 return 0
433 else
434 to.data[ptr] = a
435 end
436 to.size = max(to.size, ptr)
437 if !to.append
438 to.ptr += 1
439 end
440 return sizeof(UInt8)
441 end
442
443 readbytes!(io::GenericIOBuffer, b::Array{UInt8}, nb=length(b)) = readbytes!(io, b, Int(nb))
444 function readbytes!(io::GenericIOBuffer, b::Array{UInt8}, nb::Int)
445 nr = min(nb, bytesavailable(io))
446 if length(b) < nr
447 resize!(b, nr)
448 end
449 read_sub(io, b, 1, nr)
450 return nr
451 end
452 read(io::GenericIOBuffer) = read!(io,StringVector(bytesavailable(io)))
453 readavailable(io::GenericIOBuffer) = read(io)
454 read(io::GenericIOBuffer, nb::Integer) = read!(io,StringVector(min(nb, bytesavailable(io))))
455
456 function occursin(delim::UInt8, buf::IOBuffer)
457 p = pointer(buf.data, buf.ptr)
458 q = GC.@preserve buf ccall(:memchr,Ptr{UInt8},(Ptr{UInt8},Int32,Csize_t),p,delim,bytesavailable(buf))
459 return q != C_NULL
460 end
461
462 function occursin(delim::UInt8, buf::GenericIOBuffer)
463 data = buf.data
464 for i = buf.ptr:buf.size
465 @inbounds b = data[i]
466 b == delim && return true
467 end
468 return false
469 end
470
471 function readuntil(io::GenericIOBuffer, delim::UInt8; keep::Bool=false)
472 lb = 70
473 A = StringVector(lb)
474 nread = 0
475 nout = 0
476 data = io.data
477 for i = io.ptr : io.size
478 @inbounds b = data[i]
479 nread += 1
480 if keep || b != delim
481 nout += 1
482 if nout > lb
483 lb = nout*2
484 resize!(A, lb)
485 end
486 @inbounds A[nout] = b
487 end
488 if b == delim
489 break
490 end
491 end
492 io.ptr += nread
493 if lb != nout
494 resize!(A, nout)
495 end
496 A
497 end
498
499 # copy-free crc32c of IOBuffer:
500 function _crc32c(io::IOBuffer, nb::Integer, crc::UInt32=0x00000000)
501 nb < 0 && throw(ArgumentError("number of bytes to checksum must be ≥ 0"))
502 io.readable || throw(ArgumentError("read failed, IOBuffer is not readable"))
503 n = min(nb, bytesavailable(io))
504 n == 0 && return crc
505 crc = GC.@preserve io unsafe_crc32c(pointer(io.data, io.ptr), n, crc)
506 io.ptr += n
507 return crc
508 end
509 _crc32c(io::IOBuffer, crc::UInt32=0x00000000) = _crc32c(io, bytesavailable(io), crc)