StatProfilerHTML.jl report
Generated on tor 10 okt 2019 11:38:33
File source code
Line Exclusive Inclusive Code
1 # This file is a part of Julia. License is MIT: https://julialang.org/license
2
3 # Base.require is the implementation for the `import` statement
4
5 # Cross-platform case-sensitive path canonicalization
6
7 if Sys.isunix() && !Sys.isapple()
8 # assume case-sensitive filesystems, don't have to do anything
9 isfile_casesensitive(path) = isfile(path)
10 elseif Sys.iswindows()
11 # GetLongPathName Win32 function returns the case-preserved filename on NTFS.
12 function isfile_casesensitive(path)
13 isfile(path) || return false # Fail fast
14 basename(Filesystem.longpath(path)) == basename(path)
15 end
16 elseif Sys.isapple()
17 # HFS+ filesystem is case-preserving. The getattrlist API returns
18 # a case-preserved filename. In the rare event that HFS+ is operating
19 # in case-sensitive mode, this will still work but will be redundant.
20
21 # Constants from <sys/attr.h>
22 const ATRATTR_BIT_MAP_COUNT = 5
23 const ATTR_CMN_NAME = 1
24 const BITMAPCOUNT = 1
25 const COMMONATTR = 5
26 const FSOPT_NOFOLLOW = 1 # Don't follow symbolic links
27
28 const attr_list = zeros(UInt8, 24)
29 attr_list[BITMAPCOUNT] = ATRATTR_BIT_MAP_COUNT
30 attr_list[COMMONATTR] = ATTR_CMN_NAME
31
32 # This essentially corresponds to the following C code:
33 # attrlist attr_list;
34 # memset(&attr_list, 0, sizeof(attr_list));
35 # attr_list.bitmapcount = ATTR_BIT_MAP_COUNT;
36 # attr_list.commonattr = ATTR_CMN_NAME;
37 # struct Buffer {
38 # u_int32_t total_length;
39 # u_int32_t filename_offset;
40 # u_int32_t filename_length;
41 # char filename[max_filename_length];
42 # };
43 # Buffer buf;
44 # getattrpath(path, &attr_list, &buf, sizeof(buf), FSOPT_NOFOLLOW);
45 function isfile_casesensitive(path)
46 isfile(path) || return false
47 path_basename = String(basename(path))
48 local casepreserved_basename
49 header_size = 12
50 buf = Vector{UInt8}(undef, length(path_basename) + header_size + 1)
51 while true
52 ret = ccall(:getattrlist, Cint,
53 (Cstring, Ptr{Cvoid}, Ptr{Cvoid}, Csize_t, Culong),
54 path, attr_list, buf, sizeof(buf), FSOPT_NOFOLLOW)
55 systemerror(:getattrlist, ret ≠ 0)
56 filename_length = GC.@preserve buf unsafe_load(
57 convert(Ptr{UInt32}, pointer(buf) + 8))
58 if (filename_length + header_size) > length(buf)
59 resize!(buf, filename_length + header_size)
60 continue
61 end
62 casepreserved_basename =
63 view(buf, (header_size+1):(header_size+filename_length-1))
64 break
65 end
66 # Hack to compensate for inability to create a string from a subarray with no allocations.
67 codeunits(path_basename) == casepreserved_basename && return true
68
69 # If there is no match, it's possible that the file does exist but HFS+
70 # performed unicode normalization. See https://developer.apple.com/library/mac/qa/qa1235/_index.html.
71 isascii(path_basename) && return false
72 codeunits(Unicode.normalize(path_basename, :NFD)) == casepreserved_basename
73 end
74 else
75 # Generic fallback that performs a slow directory listing.
76 function isfile_casesensitive(path)
77 isfile(path) || return false
78 dir, filename = splitdir(path)
79 any(readdir(dir) .== filename)
80 end
81 end
82
83 ## SHA1 ##
84
85 struct SHA1
86 bytes::Vector{UInt8}
87 function SHA1(bytes::Vector{UInt8})
88 length(bytes) == 20 ||
89 throw(ArgumentError("wrong number of bytes for SHA1 hash: $(length(bytes))"))
90 return new(bytes)
91 end
92 end
93 SHA1(s::AbstractString) = SHA1(hex2bytes(s))
94
95 string(hash::SHA1) = bytes2hex(hash.bytes)
96 print(io::IO, hash::SHA1) = bytes2hex(io, hash.bytes)
97 show(io::IO, hash::SHA1) = print(io, "SHA1(\"", hash, "\")")
98
99 isless(a::SHA1, b::SHA1) = lexless(a.bytes, b.bytes)
100 hash(a::SHA1, h::UInt) = hash((SHA1, a.bytes), h)
101 ==(a::SHA1, b::SHA1) = a.bytes == b.bytes
102
103 # fake uuid5 function (for self-assigned UUIDs)
104 # TODO: delete and use real uuid5 once it's in stdlib
105
106 function uuid5(namespace::UUID, key::String)
107 u::UInt128 = 0
108 h = hash(namespace)
109 for _ = 1:sizeof(u)÷sizeof(h)
110 u <<= sizeof(h) << 3
111 u |= (h = hash(key, h))
112 end
113 u &= 0xffffffffffff0fff3fffffffffffffff
114 u |= 0x00000000000050008000000000000000
115 return UUID(u)
116 end
117
118 const ns_dummy_uuid = UUID("fe0723d6-3a44-4c41-8065-ee0f42c8ceab")
119
120 dummy_uuid(project_file::String) = isfile_casesensitive(project_file) ?
121 uuid5(ns_dummy_uuid, realpath(project_file)) : nothing
122
123 ## package path slugs: turning UUID + SHA1 into a pair of 4-byte "slugs" ##
124
125 const slug_chars = String(['A':'Z'; 'a':'z'; '0':'9'])
126
127 function slug(x::UInt32, p::Int)
128 sprint(sizehint=p) do io
129 n = length(slug_chars)
130 for i = 1:p
131 x, d = divrem(x, n)
132 write(io, slug_chars[1+d])
133 end
134 end
135 end
136
137 function package_slug(uuid::UUID, p::Int=5)
138 crc = _crc32c(uuid)
139 return slug(crc, p)
140 end
141
142 function version_slug(uuid::UUID, sha1::SHA1, p::Int=5)
143 crc = _crc32c(uuid)
144 crc = _crc32c(sha1.bytes, crc)
145 return slug(crc, p)
146 end
147
148 ## package identification: determine unique identity of package to be loaded ##
149
150 find_package(args...) = locate_package(identify_package(args...))
151
152 struct PkgId
153 uuid::Union{UUID,Nothing}
154 name::String
155
156 PkgId(u::UUID, name::AbstractString) = new(UInt128(u) == 0 ? nothing : u, name)
157 PkgId(::Nothing, name::AbstractString) = new(nothing, name)
158 end
159 PkgId(name::AbstractString) = PkgId(nothing, name)
160
161 function PkgId(m::Module, name::String = String(nameof(moduleroot(m))))
162 uuid = UUID(ccall(:jl_module_uuid, NTuple{2, UInt64}, (Any,), m))
163 UInt128(uuid) == 0 ? PkgId(name) : PkgId(uuid, name)
164 end
165
166 ==(a::PkgId, b::PkgId) = a.uuid == b.uuid && a.name == b.name
167
168 function hash(pkg::PkgId, h::UInt)
169 h += 0xc9f248583a0ca36c % UInt
170 h = hash(pkg.uuid, h)
171 h = hash(pkg.name, h)
172 return h
173 end
174
175 show(io::IO, pkg::PkgId) =
176 print(io, pkg.name, " [", pkg.uuid === nothing ? "top-level" : pkg.uuid, "]")
177
178 function binpack(pkg::PkgId)
179 io = IOBuffer()
180 write(io, UInt8(0))
181 uuid = pkg.uuid
182 write(io, uuid === nothing ? UInt128(0) : UInt128(uuid))
183 write(io, pkg.name)
184 return String(take!(io))
185 end
186
187 function binunpack(s::String)
188 io = IOBuffer(s)
189 @assert read(io, UInt8) === 0x00
190 uuid = read(io, UInt128)
191 name = read(io, String)
192 return PkgId(UUID(uuid), name)
193 end
194
195 function identify_package(where::Module, name::String)::Union{Nothing,PkgId}
196 identify_package(PkgId(where), name)
197 end
198
199 function identify_package(where::PkgId, name::String)::Union{Nothing,PkgId}
200 where.name === name && return where
201 where.uuid === nothing && return identify_package(name)
202 for env in load_path()
203 found_or_uuid = manifest_deps_get(env, where, name)
204 found_or_uuid isa UUID && return PkgId(found_or_uuid, name)
205 found_or_uuid && return nothing
206 end
207 return nothing
208 end
209
210 function identify_package(name::String)::Union{Nothing,PkgId}
211 for env in load_path()
212 found_or_uuid = project_deps_get(env, name)
213 found_or_uuid isa UUID && return PkgId(found_or_uuid, name)
214 found_or_uuid && return PkgId(name)
215 end
216 return nothing
217 end
218
219 function identify_package(name::String, names::String...)
220 pkg = identify_package(name)
221 pkg === nothing ? nothing :
222 pkg.uuid === nothing ? identify_package(names...) :
223 identify_package(pkg, names...)
224 end
225
226 function identify_package(where::PkgId, name::String, names::String...)
227 pkg = identify_package(where, name)
228 pkg === nothing ? nothing :
229 pkg.uuid === nothing ? identify_package(names...) :
230 identify_package(pkg, names...)
231 end
232
233 ## package location: given a package identity find file to load ##
234
235 function locate_package(pkg::PkgId)::Union{Nothing,String}
236 if pkg.uuid === nothing
237 for env in load_path()
238 found_or_uuid = project_deps_get(env, pkg.name)
239 found_or_uuid isa UUID &&
240 return locate_package(PkgId(found_or_uuid, pkg.name))
241 found_or_uuid && return implicit_manifest_uuid_path(env, pkg)
242 end
243 else
244 for env in load_path()
245 path = manifest_uuid_path(env, pkg)
246 path != nothing && return entry_path(path, pkg.name)
247 end
248 end
249 end
250 locate_package(::Nothing) = nothing
251
252 """
253 pathof(m::Module)
254
255 Return the path of `m.jl` file that was used to `import` module `m`,
256 or `nothing` if `m` was not imported from a package.
257
258 Use [`dirname`](@ref) to get the directory part and [`basename`](@ref)
259 to get the file name part of the path.
260 """
261 function pathof(m::Module)
262 pkgid = get(Base.module_keys, m, nothing)
263 pkgid === nothing && return nothing
264 return Base.locate_package(pkgid)
265 end
266
267 ## generic project & manifest API ##
268
269 const project_names = ("JuliaProject.toml", "Project.toml")
270 const manifest_names = ("JuliaManifest.toml", "Manifest.toml")
271
272 # return means
273 # - `false`: nothing to see here
274 # - `true`: `env` is an implicit environment
275 # - `path`: the path of an explicit project file
276 function env_project_file(env::String)::Union{Bool,String}
277 if isdir(env)
278 for proj in project_names
279 project_file = joinpath(env, proj)
280 isfile_casesensitive(project_file) && return project_file
281 end
282 return true
283 elseif basename(env) in project_names && isfile_casesensitive(env)
284 return env
285 end
286 return false
287 end
288
289 function project_deps_get(env::String, name::String)::Union{Bool,UUID}
290 project_file = env_project_file(env)
291 if project_file isa String
292 return explicit_project_deps_get(project_file, name)
293 end
294 project_file && implicit_project_deps_get(env, name)
295 end
296
297 function manifest_deps_get(env::String, where::PkgId, name::String)::Union{Bool,UUID}
298 @assert where.uuid !== nothing
299 project_file = env_project_file(env)
300 if project_file isa String
301 proj_name, proj_uuid = project_file_name_uuid_path(project_file, where.name)
302 if proj_name == where.name && proj_uuid == where.uuid
303 # `where` matches the project, use deps as manifest
304 found_or_uuid = explicit_project_deps_get(project_file, name)
305 return found_or_uuid isa UUID ? found_or_uuid : true
306 end
307 # look for `where` stanza in manifest file
308 manifest_file = project_file_manifest_path(project_file)
309 if isfile_casesensitive(manifest_file)
310 return explicit_manifest_deps_get(manifest_file, where.uuid, name)
311 end
312 return false # `where` stanza not found
313 end
314 project_file && implicit_manifest_deps_get(env, where, name)
315 end
316
317 function manifest_uuid_path(env::String, pkg::PkgId)::Union{Nothing,String}
318 project_file = env_project_file(env)
319 if project_file isa String
320 proj_name, proj_uuid, path = project_file_name_uuid_path(project_file, pkg.name)
321 proj_name == pkg.name && proj_uuid == pkg.uuid && return path
322 manifest_file = project_file_manifest_path(project_file)
323 if isfile_casesensitive(manifest_file)
324 return explicit_manifest_uuid_path(manifest_file, pkg)
325 end
326 return nothing
327 end
328 project_file ? implicit_manifest_uuid_path(env, pkg) : nothing
329 end
330
331 # regular expressions for scanning project & manifest files
332
333 const re_section = r"^\s*\["
334 const re_array_of_tables = r"^\s*\[\s*\["
335 const re_section_deps = r"^\s*\[\s*\"?deps\"?\s*\]\s*(?:#|$)"
336 const re_section_capture = r"^\s*\[\s*\[\s*\"?(\w+)\"?\s*\]\s*\]\s*(?:#|$)"
337 const re_subsection_deps = r"^\s*\[\s*\"?(\w+)\"?\s*\.\s*\"?deps\"?\s*\]\s*(?:#|$)"
338 const re_key_to_string = r"^\s*(\w+)\s*=\s*\"(.*)\"\s*(?:#|$)"
339 const re_uuid_to_string = r"^\s*uuid\s*=\s*\"(.*)\"\s*(?:#|$)"
340 const re_name_to_string = r"^\s*name\s*=\s*\"(.*)\"\s*(?:#|$)"
341 const re_path_to_string = r"^\s*path\s*=\s*\"(.*)\"\s*(?:#|$)"
342 const re_hash_to_string = r"^\s*git-tree-sha1\s*=\s*\"(.*)\"\s*(?:#|$)"
343 const re_manifest_to_string = r"^\s*manifest\s*=\s*\"(.*)\"\s*(?:#|$)"
344 const re_deps_to_any = r"^\s*deps\s*=\s*(.*?)\s*(?:#|$)"
345
346 # find project file's top-level UUID entry (or nothing)
347 function project_file_name_uuid_path(project_file::String,
348 name::String)::Tuple{String,UUID,String}
349 open(project_file) do io
350 uuid = dummy_uuid(project_file)
351 path = joinpath("src", "$name.jl")
352 for line in eachline(io)
353 occursin(re_section, line) && break
354 if (m = match(re_name_to_string, line)) != nothing
355 name = String(m.captures[1])
356 elseif (m = match(re_uuid_to_string, line)) != nothing
357 uuid = UUID(m.captures[1])
358 elseif (m = match(re_path_to_string, line)) != nothing
359 path = String(m.captures[1])
360 end
361 end
362 path = joinpath(dirname(project_file), path)
363 return name, uuid, path
364 end
365 end
366
367 # find project file's corresponding manifest file
368 function project_file_manifest_path(project_file::String)::Union{Nothing,String}
369 open(project_file) do io
370 dir = abspath(dirname(project_file))
371 for line in eachline(io)
372 occursin(re_section, line) && break
373 if (m = match(re_manifest_to_string, line)) != nothing
374 return normpath(joinpath(dir, m.captures[1]))
375 end
376 end
377 local manifest_file
378 for mfst in manifest_names
379 manifest_file = joinpath(dir, mfst)
380 isfile_casesensitive(manifest_file) && return manifest_file
381 end
382 return manifest_file
383 end
384 end
385
386 # find `name` in a manifest file and return its UUID
387 function manifest_file_name_uuid(manifest_file::String, name::String, io::IO)::Union{Nothing,UUID}
388 uuid = name′ = nothing
389 for line in eachline(io)
390 if (m = match(re_section_capture, line)) != nothing
391 name′ == name && break
392 name′ = String(m.captures[1])
393 elseif (m = match(re_uuid_to_string, line)) != nothing
394 uuid = UUID(m.captures[1])
395 end
396 end
397 name′ == name ? uuid : nothing
398 end
399
400 # given package dir and name, find an entry point
401 # and project file if one exists (or nothing if not)
402 function entry_point_and_project_file(dir::String, name::String)::Union{Tuple{Nothing,Nothing},Tuple{String,Nothing},Tuple{String,String}}
403 for entry in ("", joinpath(name, "src"), joinpath("$name.jl", "src"))
404 path = normpath(joinpath(dir, entry, "$name.jl"))
405 isfile_casesensitive(path) || continue
406 if !isempty(entry)
407 for proj in project_names
408 project_file = normpath(joinpath(dir, dirname(entry), proj))
409 isfile_casesensitive(project_file) || continue
410 return path, project_file
411 end
412 end
413 return path, nothing
414 end
415 return nothing, nothing
416 end
417
418 # given a path and a name, return the entry point
419 function entry_path(path::String, name::String)::Union{Nothing,String}
420 isfile_casesensitive(path) && return normpath(path)
421 path = normpath(joinpath(path, "src", "$name.jl"))
422 isfile_casesensitive(path) ? path : nothing
423 end
424 entry_path(::Nothing, name::String) = nothing
425
426 # given a project path (project directory or entry point)
427 # return the project file
428 function package_path_to_project_file(path::String)::Union{Nothing,String}
429 if !isdir(path)
430 dir = dirname(path)
431 basename(dir) == "src" || return nothing
432 path = dirname(dir)
433 end
434 for proj in project_names
435 project_file = joinpath(path, proj)
436 isfile_casesensitive(project_file) && return project_file
437 end
438 end
439
440 ## explicit project & manifest API ##
441
442 # find project file root or deps `name => uuid` mapping
443 # - `false` means: did not find `name`
444 # - `true` means: found `name` without UUID (can't happen in explicit projects)
445 # - `uuid` means: found `name` with `uuid` in project file
446
447 function explicit_project_deps_get(project_file::String, name::String)::Union{Bool,UUID}
448 open(project_file) do io
449 root_name = nothing
450 root_uuid = dummy_uuid(project_file)
451 state = :top
452 for line in eachline(io)
453 if state == :top
454 if occursin(re_section, line)
455 root_name == name && return root_uuid
456 state = occursin(re_section_deps, line) ? :deps : :other
457 elseif (m = match(re_name_to_string, line)) != nothing
458 root_name = String(m.captures[1])
459 elseif (m = match(re_uuid_to_string, line)) != nothing
460 root_uuid = UUID(m.captures[1])
461 end
462 elseif state == :deps
463 if (m = match(re_key_to_string, line)) != nothing
464 m.captures[1] == name && return UUID(m.captures[2])
465 end
466 end
467 if occursin(re_section, line)
468 state = occursin(re_section_deps, line) ? :deps : :other
469 end
470 end
471 return root_name == name && root_uuid
472 end
473 end
474
475 # find `where` stanza and `name` in its deps and return its UUID
476 # - `false` means: did not find `where`
477 # - `true` means: found `where` but `name` not in its deps
478 # - `uuid` means: found `where` and `name` mapped to `uuid` in its deps
479
480 function explicit_manifest_deps_get(manifest_file::String, where::UUID, name::String)::Union{Bool,UUID}
481 open(manifest_file) do io
482 uuid = deps = nothing
483 state = :other
484 for line in eachline(io)
485 if occursin(re_array_of_tables, line)
486 uuid == where && break
487 uuid = deps = nothing
488 state = :stanza
489 elseif state == :stanza
490 if (m = match(re_uuid_to_string, line)) != nothing
491 uuid = UUID(m.captures[1])
492 elseif (m = match(re_deps_to_any, line)) != nothing
493 deps = String(m.captures[1])
494 elseif occursin(re_subsection_deps, line)
495 state = :deps
496 elseif occursin(re_section, line)
497 state = :other
498 end
499 elseif state == :deps && uuid == where
500 if (m = match(re_key_to_string, line)) != nothing
501 m.captures[1] == name && return UUID(m.captures[2])
502 end
503 end
504 end
505 uuid == where || return false
506 deps === nothing && return true
507 # TODO: handle inline table syntax
508 if deps[1] != '[' || deps[end] != ']'
509 @warn "Unexpected TOML deps format:\n$deps"
510 return nothing
511 end
512 occursin(repr(name), deps) || return true
513 seekstart(io) # rewind IO handle
514 return manifest_file_name_uuid(manifest_file, name, io)
515 end
516 end
517
518 # find `uuid` stanza, return the corresponding path
519 function explicit_manifest_uuid_path(manifest_file::String, pkg::PkgId)::Union{Nothing,String}
520 open(manifest_file) do io
521 uuid = name = path = hash = nothing
522 for line in eachline(io)
523 if (m = match(re_section_capture, line)) != nothing
524 uuid == pkg.uuid && break
525 name = String(m.captures[1])
526 path = hash = nothing
527 elseif (m = match(re_uuid_to_string, line)) != nothing
528 uuid = UUID(m.captures[1])
529 elseif (m = match(re_path_to_string, line)) != nothing
530 path = String(m.captures[1])
531 elseif (m = match(re_hash_to_string, line)) != nothing
532 hash = SHA1(m.captures[1])
533 end
534 end
535 uuid == pkg.uuid || return nothing
536 name == pkg.name || return nothing # TODO: allow a mismatch?
537 if path != nothing
538 path = normpath(abspath(dirname(manifest_file), path))
539 return entry_path(path, name)
540 end
541 hash == nothing && return nothing
542 # Keep the 4 since it used to be the default
543 for slug in (version_slug(uuid, hash, 4), version_slug(uuid, hash))
544 for depot in DEPOT_PATH
545 path = abspath(depot, "packages", name, slug)
546 ispath(path) && return entry_path(path, name)
547 end
548 end
549 end
550 end
551
552 ## implicit project & manifest API ##
553
554 # look for an entry point for `name`:
555 # - `false` means: did not find `name`
556 # - `true` means: found `name` without project file
557 # - `uuid` means: found `name` with project file with real or dummy `uuid`
558 function implicit_project_deps_get(dir::String, name::String)::Union{Bool,UUID}
559 path, project_file = entry_point_and_project_file(dir, name)
560 project_file == nothing && return path != nothing
561 proj_name, proj_uuid = project_file_name_uuid_path(project_file, name)
562 proj_name == name && proj_uuid
563 end
564
565 # look for an entry-point for `where` by name, check that UUID matches
566 # if there's a project file, look up `name` in its deps and return that
567 # - `false` means: did not find `where`
568 # - `true` means: found `where` but `name` not in its deps
569 # - `uuid` means: found `where` and `name` mapped to `uuid` in its deps
570 function implicit_manifest_deps_get(dir::String, where::PkgId, name::String)::Union{Bool,UUID}
571 @assert where.uuid !== nothing
572 project_file = entry_point_and_project_file(dir, where.name)[2]
573 project_file === nothing && return false
574 proj_name, proj_uuid = project_file_name_uuid_path(project_file, where.name)
575 proj_name == where.name && proj_uuid == where.uuid || return false
576 found_or_uuid = explicit_project_deps_get(project_file, name)
577 found_or_uuid isa UUID ? found_or_uuid : true
578 end
579
580 # look for an entry-point for `pkg` and return its path if UUID matches
581 function implicit_manifest_uuid_path(dir::String, pkg::PkgId)::Union{Nothing,String}
582 path, project_file = entry_point_and_project_file(dir, pkg.name)
583 pkg.uuid === nothing && project_file === nothing && return path
584 pkg.uuid === nothing || project_file === nothing && return nothing
585 proj_name, proj_uuid = project_file_name_uuid_path(project_file, pkg.name)
586 proj_name == pkg.name && proj_uuid == pkg.uuid ? path : nothing
587 end
588
589 ## other code loading functionality ##
590
591 function find_source_file(path::AbstractString)
592 (isabspath(path) || isfile(path)) && return path
593 base_path = joinpath(Sys.BINDIR::String, DATAROOTDIR, "julia", "base", path)
594 return isfile(base_path) ? base_path : nothing
595 end
596
597 cache_file_entry(pkg::PkgId) = joinpath(
598 "compiled",
599 "v$(VERSION.major).$(VERSION.minor)",
600 pkg.uuid === nothing ? "$(pkg.name).ji" : joinpath(pkg.name, "$(package_slug(pkg.uuid)).ji")
601 )
602
603 function find_all_in_cache_path(pkg::PkgId)
604 paths = String[]
605 entry = cache_file_entry(pkg)
606 for depot in DEPOT_PATH
607 path = joinpath(depot, entry)
608 isfile_casesensitive(path) && push!(paths, path)
609 end
610 return paths
611 end
612
613 # these return either the array of modules loaded from the path / content given
614 # or an Exception that describes why it couldn't be loaded
615 # and it reconnects the Base.Docs.META
616 function _include_from_serialized(path::String, depmods::Vector{Any})
617 sv = ccall(:jl_restore_incremental, Any, (Cstring, Any), path, depmods)
618 if isa(sv, Exception)
619 return sv
620 end
621 restored = sv[1]
622 if !isa(restored, Exception)
623 for M in restored::Vector{Any}
624 M = M::Module
625 if isdefined(M, Base.Docs.META)
626 push!(Base.Docs.modules, M)
627 end
628 if parentmodule(M) === M
629 register_root_module(M)
630 end
631 end
632 end
633 isassigned(sv, 2) && ccall(:jl_init_restored_modules, Cvoid, (Any,), sv[2])
634 return restored
635 end
636
637 function _tryrequire_from_serialized(modkey::PkgId, build_id::UInt64, modpath::Union{Nothing, String})
638 if root_module_exists(modkey)
639 M = root_module(modkey)
640 if PkgId(M) == modkey && module_build_id(M) === build_id
641 return M
642 end
643 else
644 if modpath === nothing
645 modpath = locate_package(modkey)
646 modpath === nothing && return nothing
647 end
648 mod = _require_search_from_serialized(modkey, String(modpath))
649 if !isa(mod, Bool)
650 for callback in package_callbacks
651 invokelatest(callback, modkey)
652 end
653 for M in mod::Vector{Any}
654 if PkgId(M) == modkey && module_build_id(M) === build_id
655 return M
656 end
657 end
658 end
659 end
660 return nothing
661 end
662
663 function _require_from_serialized(path::String)
664 # loads a precompile cache file, ignoring stale_cachfile tests
665 # load all of the dependent modules first
666 local depmodnames
667 io = open(path, "r")
668 try
669 isvalid_cache_header(io) || return ArgumentError("Invalid header in cache file $path.")
670 depmodnames = parse_cache_header(io)[3]
671 isvalid_file_crc(io) || return ArgumentError("Invalid checksum in cache file $path.")
672 finally
673 close(io)
674 end
675 ndeps = length(depmodnames)
676 depmods = Vector{Any}(undef, ndeps)
677 for i in 1:ndeps
678 modkey, build_id = depmodnames[i]
679 dep = _tryrequire_from_serialized(modkey, build_id, nothing)
680 dep === nothing && return ErrorException("Required dependency $modkey failed to load from a cache file.")
681 depmods[i] = dep::Module
682 end
683 # then load the file
684 return _include_from_serialized(path, depmods)
685 end
686
687 # returns `true` if require found a precompile cache for this sourcepath, but couldn't load it
688 # returns `false` if the module isn't known to be precompilable
689 # returns the set of modules restored if the cache load succeeded
690 function _require_search_from_serialized(pkg::PkgId, sourcepath::String)
691 paths = find_all_in_cache_path(pkg)
692 for path_to_try in paths::Vector{String}
693 staledeps = stale_cachefile(sourcepath, path_to_try)
694 if staledeps === true
695 continue
696 end
697 # finish loading module graph into staledeps
698 for i in 1:length(staledeps)
699 dep = staledeps[i]
700 dep isa Module && continue
701 modpath, modkey, build_id = dep::Tuple{String, PkgId, UInt64}
702 dep = _tryrequire_from_serialized(modkey, build_id, modpath)
703 if dep === nothing
704 @debug "Required dependency $modkey failed to load from cache file for $modpath."
705 staledeps = true
706 break
707 end
708 staledeps[i] = dep::Module
709 end
710 if staledeps === true
711 continue
712 end
713 restored = _include_from_serialized(path_to_try, staledeps)
714 if isa(restored, Exception)
715 @debug "Deserialization checks failed while attempting to load cache from $path_to_try" exception=restored
716 else
717 return restored
718 end
719 end
720 return !isempty(paths)
721 end
722
723 # to synchronize multiple tasks trying to import/using something
724 const package_locks = Dict{PkgId,Condition}()
725
726 # to notify downstream consumers that a module was successfully loaded
727 # Callbacks take the form (mod::Base.PkgId) -> nothing.
728 # WARNING: This is an experimental feature and might change later, without deprecation.
729 const package_callbacks = Any[]
730 # to notify downstream consumers that a file has been included into a particular module
731 # Callbacks take the form (mod::Module, filename::String) -> nothing
732 # WARNING: This is an experimental feature and might change later, without deprecation.
733 const include_callbacks = Any[]
734
735 # used to optionally track dependencies when requiring a module:
736 const _concrete_dependencies = Pair{PkgId,UInt64}[] # these dependency versions are "set in stone", and the process should try to avoid invalidating them
737 const _require_dependencies = Any[] # a list of (mod, path, mtime) tuples that are the file dependencies of the module currently being precompiled
738 const _track_dependencies = Ref(false) # set this to true to track the list of file dependencies
739 function _include_dependency(mod::Module, _path::AbstractString)
740 prev = source_path(nothing)
741 if prev === nothing
742 path = abspath(_path)
743 else
744 path = normpath(joinpath(dirname(prev), _path))
745 end
746 if _track_dependencies[]
747 push!(_require_dependencies, (mod, path, mtime(path)))
748 end
749 return path, prev
750 end
751
752 """
753 include_dependency(path::AbstractString)
754
755 In a module, declare that the file specified by `path` (relative or absolute) is a
756 dependency for precompilation; that is, the module will need to be recompiled if this file
757 changes.
758
759 This is only needed if your module depends on a file that is not used via `include`. It has
760 no effect outside of compilation.
761 """
762 function include_dependency(path::AbstractString)
763 _include_dependency(Main, path)
764 return nothing
765 end
766
767 # we throw PrecompilableError when a module doesn't want to be precompiled
768 struct PrecompilableError <: Exception end
769 function show(io::IO, ex::PrecompilableError)
770 print(io, "Declaring __precompile__(false) is not allowed in files that are being precompiled.")
771 end
772 precompilableerror(ex::PrecompilableError) = true
773 precompilableerror(ex::WrappedException) = precompilableerror(ex.error)
774 precompilableerror(@nospecialize ex) = false
775
776 # Call __precompile__(false) at the top of a tile prevent it from being precompiled (false)
777 """
778 __precompile__(isprecompilable::Bool)
779
780 Specify whether the file calling this function is precompilable, defaulting to `true`.
781 If a module or file is *not* safely precompilable, it should call `__precompile__(false)` in
782 order to throw an error if Julia attempts to precompile it.
783 """
784 @noinline function __precompile__(isprecompilable::Bool=true)
785 if !isprecompilable && ccall(:jl_generating_output, Cint, ()) != 0
786 throw(PrecompilableError())
787 end
788 nothing
789 end
790
791 # require always works in Main scope and loads files from node 1
792 const toplevel_load = Ref(true)
793
794 const full_warning_showed = Ref(false)
795 const modules_warned_for = Set{PkgId}()
796
797 """
798 require(into::Module, module::Symbol)
799
800 This function is part of the implementation of `using` / `import`, if a module is not
801 already defined in `Main`. It can also be called directly to force reloading a module,
802 regardless of whether it has been loaded before (for example, when interactively developing
803 libraries).
804
805 Loads a source file, in the context of the `Main` module, on every active node, searching
806 standard locations for files. `require` is considered a top-level operation, so it sets the
807 current `include` path but does not use it to search for files (see help for `include`).
808 This function is typically used to load library code, and is implicitly called by `using` to
809 load packages.
810
811 When searching for files, `require` first looks for package code in the global array
812 `LOAD_PATH`. `require` is case-sensitive on all platforms, including those with
813 case-insensitive filesystems like macOS and Windows.
814
815 For more details regarding code loading, see the manual.
816 """
817 function require(into::Module, mod::Symbol)
818 uuidkey = identify_package(into, String(mod))
819 # Core.println("require($(PkgId(into)), $mod) -> $uuidkey")
820 if uuidkey === nothing
821 where = PkgId(into)
822 if where.uuid === nothing
823 throw(ArgumentError("""
824 Package $mod not found in current path:
825 - Run `import Pkg; Pkg.add($(repr(String(mod))))` to install the $mod package.
826 """))
827 else
828 s = """
829 Package $(where.name) does not have $mod in its dependencies:
830 - If you have $(where.name) checked out for development and have
831 added $mod as a dependency but haven't updated your primary
832 environment's manifest file, try `Pkg.resolve()`.
833 - Otherwise you may need to report an issue with $(where.name)"""
834
835 uuidkey = identify_package(PkgId(string(into)), String(mod))
836 uuidkey === nothing && throw(ArgumentError(s))
837
838 # fall back to toplevel loading with a warning
839 if !(where in modules_warned_for)
840 @warn string(
841 full_warning_showed[] ? "" : s, "\n",
842 string("Loading $(mod) into $(where.name) from project dependency, ",
843 "future warnings for $(where.name) are suppressed.")
844 ) _module = nothing _file = nothing _group = nothing
845 push!(modules_warned_for, where)
846 end
847 full_warning_showed[] = true
848 end
849 end
850 if _track_dependencies[]
851 push!(_require_dependencies, (into, binpack(uuidkey), 0.0))
852 end
853 return require(uuidkey)
854 end
855
856 function require(uuidkey::PkgId)
857 if !root_module_exists(uuidkey)
858 _require(uuidkey)
859 # After successfully loading, notify downstream consumers
860 for callback in package_callbacks
861 invokelatest(callback, uuidkey)
862 end
863 end
864 return root_module(uuidkey)
865 end
866
867 const loaded_modules = Dict{PkgId,Module}()
868 const module_keys = IdDict{Module,PkgId}() # the reverse
869
870 is_root_module(m::Module) = haskey(module_keys, m)
871 root_module_key(m::Module) = module_keys[m]
872
873 function register_root_module(m::Module)
874 key = PkgId(m, String(nameof(m)))
875 if haskey(loaded_modules, key)
876 oldm = loaded_modules[key]
877 if oldm !== m
878 @warn "Replacing module `$(key.name)`"
879 end
880 end
881 loaded_modules[key] = m
882 module_keys[m] = key
883 nothing
884 end
885
886 register_root_module(Core)
887 register_root_module(Base)
888 register_root_module(Main)
889
890 # This is used as the current module when loading top-level modules.
891 # It has the special behavior that modules evaluated in it get added
892 # to the loaded_modules table instead of getting bindings.
893 baremodule __toplevel__
894 using Base
895 end
896
897 # get a top-level Module from the given key
898 root_module(key::PkgId) = loaded_modules[key]
899 root_module(where::Module, name::Symbol) =
900 root_module(identify_package(where, String(name)))
901
902 root_module_exists(key::PkgId) = haskey(loaded_modules, key)
903 loaded_modules_array() = collect(values(loaded_modules))
904
905 function unreference_module(key::PkgId)
906 if haskey(loaded_modules, key)
907 m = pop!(loaded_modules, key)
908 # need to ensure all modules are GC rooted; will still be referenced
909 # in module_keys
910 end
911 end
912
913 function _require(pkg::PkgId)
914 # handle recursive calls to require
915 loading = get(package_locks, pkg, false)
916 if loading !== false
917 # load already in progress for this module
918 wait(loading)
919 return
920 end
921 package_locks[pkg] = Condition()
922
923 last = toplevel_load[]
924 try
925 toplevel_load[] = false
926 # perform the search operation to select the module file require intends to load
927 path = locate_package(pkg)
928 if path === nothing
929 throw(ArgumentError("""
930 Package $pkg is required but does not seem to be installed:
931 - Run `Pkg.instantiate()` to install all recorded dependencies.
932 """))
933 end
934
935 # attempt to load the module file via the precompile cache locations
936 if JLOptions().use_compiled_modules != 0
937 m = _require_search_from_serialized(pkg, path)
938 if !isa(m, Bool)
939 return
940 end
941 end
942
943 # if the module being required was supposed to have a particular version
944 # but it was not handled by the precompile loader, complain
945 for (concrete_pkg, concrete_build_id) in _concrete_dependencies
946 if pkg == concrete_pkg
947 @warn """Module $(pkg.name) with build ID $concrete_build_id is missing from the cache.
948 This may mean $pkg does not support precompilation but is imported by a module that does."""
949 if JLOptions().incremental != 0
950 # during incremental precompilation, this should be fail-fast
951 throw(PrecompilableError())
952 end
953 end
954 end
955
956 if JLOptions().use_compiled_modules != 0
957 if (0 == ccall(:jl_generating_output, Cint, ())) || (JLOptions().incremental != 0)
958 # spawn off a new incremental pre-compile task for recursive `require` calls
959 # or if the require search declared it was pre-compiled before (and therefore is expected to still be pre-compilable)
960 cachefile = compilecache(pkg, path)
961 if isa(cachefile, Exception)
962 if !precompilableerror(cachefile)
963 @warn "The call to compilecache failed to create a usable precompiled cache file for $pkg" exception=m
964 end
965 # fall-through to loading the file locally
966 else
967 m = _require_from_serialized(cachefile)
968 if isa(m, Exception)
969 @warn "The call to compilecache failed to create a usable precompiled cache file for $pkg" exception=m
970 else
971 return
972 end
973 end
974 end
975 end
976
977 # just load the file normally via include
978 # for unknown dependencies
979 uuid = pkg.uuid
980 uuid = (uuid === nothing ? (UInt64(0), UInt64(0)) : convert(NTuple{2, UInt64}, uuid))
981 old_uuid = ccall(:jl_module_uuid, NTuple{2, UInt64}, (Any,), __toplevel__)
982 if uuid !== old_uuid
983 ccall(:jl_set_module_uuid, Cvoid, (Any, NTuple{2, UInt64}), __toplevel__, uuid)
984 end
985 try
986 include_relative(__toplevel__, path)
987 return
988 finally
989 if uuid !== old_uuid
990 ccall(:jl_set_module_uuid, Cvoid, (Any, NTuple{2, UInt64}), __toplevel__, old_uuid)
991 end
992 end
993 finally
994 toplevel_load[] = last
995 loading = pop!(package_locks, pkg)
996 notify(loading, all=true)
997 end
998 nothing
999 end
1000
1001 # relative-path load
1002
1003 """
1004 include_string(m::Module, code::AbstractString, filename::AbstractString="string")
1005
1006 Like `include`, except reads code from the given string rather than from a file.
1007 """
1008 include_string(m::Module, txt::String, fname::String) =
1009 ccall(:jl_load_file_string, Any, (Ptr{UInt8}, Csize_t, Cstring, Any),
1010 txt, sizeof(txt), fname, m)
1011
1012 include_string(m::Module, txt::AbstractString, fname::AbstractString="string") =
1013 include_string(m, String(txt), String(fname))
1014
1015 function source_path(default::Union{AbstractString,Nothing}="")
1016 t = current_task()
1017 while true
1018 s = t.storage
1019 if s !== nothing && haskey(s, :SOURCE_PATH)
1020 return s[:SOURCE_PATH]
1021 end
1022 if t === t.parent
1023 return default
1024 end
1025 t = t.parent
1026 end
1027 end
1028
1029 function source_dir()
1030 p = source_path(nothing)
1031 p === nothing ? pwd() : dirname(p)
1032 end
1033
1034 include_relative(mod::Module, path::AbstractString) = include_relative(mod, String(path))
1035
40 (95.24%) samples spent in include_relative
0 (ex.), 40 (100.00%) (incl.) when called from include line 29
function include_relative(mod::Module, _path::String)
1036 path, prev = _include_dependency(mod, _path)
1037 for callback in include_callbacks # to preserve order, must come before Core.include
1038 invokelatest(callback, mod, path)
1039 end
1040 tls = task_local_storage()
1041 tls[:SOURCE_PATH] = path
1042 local result
1043 try
1044 40 (95.24%)
40 (100.00%) samples spent calling include
result = Core.include(mod, path)
1045 finally
1046 if prev === nothing
1047 delete!(tls, :SOURCE_PATH)
1048 else
1049 tls[:SOURCE_PATH] = prev
1050 end
1051 end
1052 return result
1053 end
1054
1055 """
1056 Base.include([m::Module,] path::AbstractString)
1057
1058 Evaluate the contents of the input source file in the global scope of module `m`.
1059 Every module (except those defined with `baremodule`) has its own 1-argument
1060 definition of `include`, which evaluates the file in that module.
1061 Returns the result of the last evaluated expression of the input file. During including,
1062 a task-local include path is set to the directory containing the file. Nested calls to
1063 `include` will search relative to that path. This function is typically used to load source
1064 interactively, or to combine files in packages that are broken into multiple source files.
1065 """
1066 Base.include # defined in sysimg.jl
1067
1068 """
1069 evalfile(path::AbstractString, args::Vector{String}=String[])
1070
1071 Load the file using [`Base.include`](@ref), evaluate all expressions,
1072 and return the value of the last one.
1073 """
1074 function evalfile(path::AbstractString, args::Vector{String}=String[])
1075 return Core.eval(Module(:__anon__),
1076 Expr(:toplevel,
1077 :(const ARGS = $args),
1078 :(eval(x) = $(Expr(:core, :eval))(__anon__, x)),
1079 :(include(x) = $(Expr(:top, :include))(__anon__, x)),
1080 :(include($path))))
1081 end
1082 evalfile(path::AbstractString, args::Vector) = evalfile(path, String[args...])
1083
1084 function load_path_setup_code(load_path::Bool=true)
1085 code = """
1086 append!(empty!(Base.DEPOT_PATH), $(repr(map(abspath, DEPOT_PATH))))
1087 append!(empty!(Base.DL_LOAD_PATH), $(repr(map(abspath, DL_LOAD_PATH))))
1088 """
1089 if load_path
1090 load_path = map(abspath, Base.load_path())
1091 path_sep = Sys.iswindows() ? ';' : ':'
1092 any(path -> path_sep in path, load_path) &&
1093 error("LOAD_PATH entries cannot contain $(repr(path_sep))")
1094 code *= """
1095 append!(empty!(Base.LOAD_PATH), $(repr(load_path)))
1096 ENV["JULIA_LOAD_PATH"] = $(repr(join(load_path, Sys.iswindows() ? ';' : ':')))
1097 Base.HOME_PROJECT[] = Base.ACTIVE_PROJECT[] = nothing
1098 """
1099 end
1100 return code
1101 end
1102
1103 function create_expr_cache(input::String, output::String, concrete_deps::typeof(_concrete_dependencies), uuid::Union{Nothing,UUID})
1104 rm(output, force=true) # Remove file if it exists
1105 code_object = """
1106 while !eof(stdin)
1107 code = readuntil(stdin, '\\0')
1108 eval(Meta.parse(code))
1109 end
1110 """
1111 io = open(pipeline(detach(`$(julia_cmd()) -O0
1112 --output-ji $output --output-incremental=yes
1113 --startup-file=no --history-file=no --warn-overwrite=yes
1114 --color=$(have_color ? "yes" : "no")
1115 --eval $code_object`), stderr=stderr),
1116 "w", stdout)
1117 in = io.in
1118 try
1119 write(in, """
1120 begin
1121 $(Base.load_path_setup_code())
1122 Base._track_dependencies[] = true
1123 Base.empty!(Base._concrete_dependencies)
1124 """)
1125 for (pkg, build_id) in concrete_deps
1126 pkg_str = if pkg.uuid === nothing
1127 "Base.PkgId($(repr(pkg.name)))"
1128 else
1129 "Base.PkgId(Base.UUID(\"$(pkg.uuid)\"), $(repr(pkg.name)))"
1130 end
1131 write(in, "Base.push!(Base._concrete_dependencies, $pkg_str => $(repr(build_id)))\n")
1132 end
1133 write(io, "end\0")
1134 uuid_tuple = uuid === nothing ? (0, 0) : convert(NTuple{2, UInt64}, uuid)
1135 write(in, "ccall(:jl_set_module_uuid, Cvoid, (Any, NTuple{2, UInt64}), Base.__toplevel__, $uuid_tuple)\0")
1136 source = source_path(nothing)
1137 if source !== nothing
1138 write(in, "task_local_storage()[:SOURCE_PATH] = $(repr(source))\0")
1139 end
1140 write(in, """
1141 try
1142 Base.include(Base.__toplevel__, $(repr(abspath(input))))
1143 catch ex
1144 Base.precompilableerror(ex) || Base.rethrow(ex)
1145 Base.@debug "Aborting `createexprcache'" exception=(Base.ErrorException("Declaration of __precompile__(false) not allowed"), Base.catch_backtrace())
1146 Base.exit(125) # we define status = 125 means PrecompileableError
1147 end\0""")
1148 # TODO: cleanup is probably unnecessary here
1149 if source !== nothing
1150 write(in, "delete!(task_local_storage(), :SOURCE_PATH)\0")
1151 end
1152 write(in, "ccall(:jl_set_module_uuid, Cvoid, (Any, NTuple{2, UInt64}), Base.__toplevel__, (0, 0))\0")
1153 close(in)
1154 catch ex
1155 close(in)
1156 process_running(io) && Timer(t -> kill(io), 5.0) # wait a short time before killing the process to give it a chance to clean up on its own first
1157 rethrow(ex)
1158 end
1159 return io
1160 end
1161
1162 """
1163 Base.compilecache(module::PkgId)
1164
1165 Creates a precompiled cache file for a module and all of its dependencies.
1166 This can be used to reduce package load times. Cache files are stored in
1167 `DEPOT_PATH[1]/compiled`. See [Module initialization and precompilation](@ref)
1168 for important notes.
1169 """
1170 function compilecache(pkg::PkgId)
1171 path = locate_package(pkg)
1172 path === nothing && throw(ArgumentError("$pkg not found during precompilation"))
1173 return compilecache(pkg, path)
1174 end
1175 function compilecache(pkg::PkgId, path::String)
1176 # decide where to put the resulting cache file
1177 cachefile = abspath(DEPOT_PATH[1], cache_file_entry(pkg))
1178 cachepath = dirname(cachefile)
1179 isdir(cachepath) || mkpath(cachepath)
1180 # build up the list of modules that we want the precompile process to preserve
1181 concrete_deps = copy(_concrete_dependencies)
1182 for (key, mod) in loaded_modules
1183 if !(mod === Main || mod === Core || mod === Base)
1184 push!(concrete_deps, key => module_build_id(mod))
1185 end
1186 end
1187 # run the expression and cache the result
1188 verbosity = isinteractive() ? CoreLogging.Info : CoreLogging.Debug
1189 if isfile(cachefile)
1190 @logmsg verbosity "Recompiling stale cache file $cachefile for $pkg"
1191 else
1192 @logmsg verbosity "Precompiling $pkg"
1193 end
1194 p = create_expr_cache(path, cachefile, concrete_deps, pkg.uuid)
1195 if success(p)
1196 # append checksum to the end of the .ji file:
1197 open(cachefile, "a+") do f
1198 write(f, _crc32c(seekstart(f)))
1199 end
1200 elseif p.exitcode == 125
1201 return PrecompilableError()
1202 else
1203 error("Failed to precompile $pkg to $cachefile.")
1204 end
1205 return cachefile
1206 end
1207
1208 module_build_id(m::Module) = ccall(:jl_module_build_id, UInt64, (Any,), m)
1209
1210 isvalid_cache_header(f::IOStream) = (0 != ccall(:jl_read_verify_header, Cint, (Ptr{Cvoid},), f.ios))
1211 isvalid_file_crc(f::IOStream) = (_crc32c(seekstart(f), filesize(f) - 4) == read(f, UInt32))
1212
1213 function parse_cache_header(f::IO)
1214 modules = Vector{Pair{PkgId, UInt64}}()
1215 while true
1216 n = read(f, Int32)
1217 n == 0 && break
1218 sym = String(read(f, n)) # module name
1219 uuid = UUID((read(f, UInt64), read(f, UInt64))) # pkg UUID
1220 build_id = read(f, UInt64) # build UUID (mostly just a timestamp)
1221 push!(modules, PkgId(uuid, sym) => build_id)
1222 end
1223 totbytes = read(f, Int64) # total bytes for file dependencies
1224 # read the list of requirements
1225 # and split the list into include and requires statements
1226 includes = Tuple{PkgId, String, Float64}[]
1227 requires = Pair{PkgId, PkgId}[]
1228 while true
1229 n2 = read(f, Int32)
1230 n2 == 0 && break
1231 depname = String(read(f, n2))
1232 mtime = read(f, Float64)
1233 n1 = read(f, Int32)
1234 # map ids to keys
1235 modkey = (n1 == 0) ? PkgId("") : modules[n1].first
1236 if n1 != 0
1237 # consume (and ignore) the module path too
1238 while true
1239 n1 = read(f, Int32)
1240 totbytes -= 4
1241 n1 == 0 && break
1242 skip(f, n1) # String(read(f, n1))
1243 totbytes -= n1
1244 end
1245 end
1246 if depname[1] == '\0'
1247 push!(requires, modkey => binunpack(depname))
1248 else
1249 push!(includes, (modkey, depname, mtime))
1250 end
1251 totbytes -= 4 + 4 + n2 + 8
1252 end
1253 @assert totbytes == 12 "header of cache file appears to be corrupt"
1254 srctextpos = read(f, Int64)
1255 # read the list of modules that are required to be present during loading
1256 required_modules = Vector{Pair{PkgId, UInt64}}()
1257 while true
1258 n = read(f, Int32)
1259 n == 0 && break
1260 sym = String(read(f, n)) # module name
1261 uuid = UUID((read(f, UInt64), read(f, UInt64))) # pkg UUID
1262 build_id = read(f, UInt64) # build id
1263 push!(required_modules, PkgId(uuid, sym) => build_id)
1264 end
1265 return modules, (includes, requires), required_modules, srctextpos
1266 end
1267
1268 function parse_cache_header(cachefile::String)
1269 io = open(cachefile, "r")
1270 try
1271 !isvalid_cache_header(io) && throw(ArgumentError("Invalid header in cache file $cachefile."))
1272 return parse_cache_header(io)
1273 finally
1274 close(io)
1275 end
1276 end
1277
1278 function cache_dependencies(f::IO)
1279 defs, (includes, requires), modules = parse_cache_header(f)
1280 return modules, map(mod_fl_mt -> (mod_fl_mt[2], mod_fl_mt[3]), includes) # discard the module
1281 end
1282
1283 function cache_dependencies(cachefile::String)
1284 io = open(cachefile, "r")
1285 try
1286 !isvalid_cache_header(io) && throw(ArgumentError("Invalid header in cache file $cachefile."))
1287 return cache_dependencies(io)
1288 finally
1289 close(io)
1290 end
1291 end
1292
1293 function read_dependency_src(io::IO, filename::AbstractString)
1294 modules, (includes, requires), required_modules, srctextpos = parse_cache_header(io)
1295 srctextpos == 0 && error("no source-text stored in cache file")
1296 seek(io, srctextpos)
1297 return _read_dependency_src(io, filename)
1298 end
1299
1300 function _read_dependency_src(io::IO, filename::AbstractString)
1301 while !eof(io)
1302 filenamelen = read(io, Int32)
1303 filenamelen == 0 && break
1304 fn = String(read(io, filenamelen))
1305 len = read(io, UInt64)
1306 if fn == filename
1307 return String(read(io, len))
1308 end
1309 seek(io, position(io) + len)
1310 end
1311 error(filename, " is not stored in the source-text cache")
1312 end
1313
1314 function read_dependency_src(cachefile::String, filename::AbstractString)
1315 io = open(cachefile, "r")
1316 try
1317 !isvalid_cache_header(io) && throw(ArgumentError("Invalid header in cache file $cachefile."))
1318 return read_dependency_src(io, filename)
1319 finally
1320 close(io)
1321 end
1322 end
1323
1324 # returns true if it "cachefile.ji" is stale relative to "modpath.jl"
1325 # otherwise returns the list of dependencies to also check
1326 function stale_cachefile(modpath::String, cachefile::String)
1327 io = open(cachefile, "r")
1328 try
1329 if !isvalid_cache_header(io)
1330 @debug "Rejecting cache file $cachefile due to it containing an invalid cache header"
1331 return true # invalid cache file
1332 end
1333 (modules, (includes, requires), required_modules) = parse_cache_header(io)
1334 modules = Dict{PkgId, UInt64}(modules)
1335
1336 # Check if transitive dependencies can be fulfilled
1337 ndeps = length(required_modules)
1338 depmods = Vector{Any}(undef, ndeps)
1339 for i in 1:ndeps
1340 req_key, req_build_id = required_modules[i]
1341 # Module is already loaded
1342 if root_module_exists(req_key)
1343 M = root_module(req_key)
1344 if PkgId(M) == req_key && module_build_id(M) === req_build_id
1345 depmods[i] = M
1346 else
1347 @debug "Rejecting cache file $cachefile because module $req_key is already loaded and incompatible."
1348 return true # Won't be able to fulfill dependency
1349 end
1350 else
1351 path = locate_package(req_key)
1352 if path === nothing
1353 @debug "Rejecting cache file $cachefile because dependency $req_key not found."
1354 return true # Won't be able to fulfill dependency
1355 end
1356 depmods[i] = (path, req_key, req_build_id)
1357 end
1358 end
1359
1360 # check if this file is going to provide one of our concrete dependencies
1361 # or if it provides a version that conflicts with our concrete dependencies
1362 # or neither
1363 skip_timecheck = false
1364 for (req_key, req_build_id) in _concrete_dependencies
1365 build_id = get(modules, req_key, UInt64(0))
1366 if build_id !== UInt64(0)
1367 if build_id === req_build_id
1368 skip_timecheck = true
1369 break
1370 end
1371 @debug "Rejecting cache file $cachefile because it provides the wrong uuid (got $build_id) for $mod (want $req_build_id)"
1372 return true # cachefile doesn't provide the required version of the dependency
1373 end
1374 end
1375
1376 # now check if this file is fresh relative to its source files
1377 if !skip_timecheck
1378 if !samefile(includes[1][2], modpath)
1379 @debug "Rejecting cache file $cachefile because it is for file $(includes[1][2])) not file $modpath"
1380 return true # cache file was compiled from a different path
1381 end
1382 for (modkey, req_modkey) in requires
1383 # verify that `require(modkey, name(req_modkey))` ==> `req_modkey`
1384 if identify_package(modkey, req_modkey.name) != req_modkey
1385 @debug "Rejecting cache file $cachefile because uuid mapping for $modkey => $req_modkey has changed"
1386 return true
1387 end
1388 end
1389 for (_, f, ftime_req) in includes
1390 # Issue #13606: compensate for Docker images rounding mtimes
1391 # Issue #20837: compensate for GlusterFS truncating mtimes to microseconds
1392 ftime = mtime(f)
1393 if ftime != ftime_req && ftime != floor(ftime_req) && ftime != trunc(ftime_req, digits=6)
1394 @debug "Rejecting stale cache file $cachefile (mtime $ftime_req) because file $f (mtime $ftime) has changed"
1395 return true
1396 end
1397 end
1398 end
1399
1400 if !isvalid_file_crc(io)
1401 @debug "Rejecting cache file $cachefile because it has an invalid checksum"
1402 return true
1403 end
1404
1405 return depmods # fresh cachefile
1406 finally
1407 close(io)
1408 end
1409 end
1410
1411 """
1412 @__FILE__ -> AbstractString
1413
1414 Expand to a string with the path to the file containing the
1415 macrocall, or an empty string if evaluated by `julia -e <expr>`.
1416 Return `nothing` if the macro was missing parser source information.
1417 Alternatively see [`PROGRAM_FILE`](@ref).
1418 """
1419 macro __FILE__()
1420 __source__.file === nothing && return nothing
1421 return String(__source__.file)
1422 end
1423
1424 """
1425 @__DIR__ -> AbstractString
1426
1427 Expand to a string with the absolute path to the directory of the file
1428 containing the macrocall.
1429 Return the current working directory if run from a REPL or if evaluated by `julia -e <expr>`.
1430 """
1431 macro __DIR__()
1432 __source__.file === nothing && return nothing
1433 return abspath(dirname(String(__source__.file)))
1434 end