Line | Exclusive | Inclusive | Code |
---|---|---|---|
1 | # This file is a part of Julia. License is MIT: https://julialang.org/license | ||
2 | |||
3 | # Base.require is the implementation for the `import` statement | ||
4 | |||
5 | # Cross-platform case-sensitive path canonicalization | ||
6 | |||
7 | if Sys.isunix() && !Sys.isapple() | ||
8 | # assume case-sensitive filesystems, don't have to do anything | ||
9 | isfile_casesensitive(path) = isfile(path) | ||
10 | elseif Sys.iswindows() | ||
11 | # GetLongPathName Win32 function returns the case-preserved filename on NTFS. | ||
12 | function isfile_casesensitive(path) | ||
13 | isfile(path) || return false # Fail fast | ||
14 | basename(Filesystem.longpath(path)) == basename(path) | ||
15 | end | ||
16 | elseif Sys.isapple() | ||
17 | # HFS+ filesystem is case-preserving. The getattrlist API returns | ||
18 | # a case-preserved filename. In the rare event that HFS+ is operating | ||
19 | # in case-sensitive mode, this will still work but will be redundant. | ||
20 | |||
21 | # Constants from <sys/attr.h> | ||
22 | const ATRATTR_BIT_MAP_COUNT = 5 | ||
23 | const ATTR_CMN_NAME = 1 | ||
24 | const BITMAPCOUNT = 1 | ||
25 | const COMMONATTR = 5 | ||
26 | const FSOPT_NOFOLLOW = 1 # Don't follow symbolic links | ||
27 | |||
28 | const attr_list = zeros(UInt8, 24) | ||
29 | attr_list[BITMAPCOUNT] = ATRATTR_BIT_MAP_COUNT | ||
30 | attr_list[COMMONATTR] = ATTR_CMN_NAME | ||
31 | |||
32 | # This essentially corresponds to the following C code: | ||
33 | # attrlist attr_list; | ||
34 | # memset(&attr_list, 0, sizeof(attr_list)); | ||
35 | # attr_list.bitmapcount = ATTR_BIT_MAP_COUNT; | ||
36 | # attr_list.commonattr = ATTR_CMN_NAME; | ||
37 | # struct Buffer { | ||
38 | # u_int32_t total_length; | ||
39 | # u_int32_t filename_offset; | ||
40 | # u_int32_t filename_length; | ||
41 | # char filename[max_filename_length]; | ||
42 | # }; | ||
43 | # Buffer buf; | ||
44 | # getattrpath(path, &attr_list, &buf, sizeof(buf), FSOPT_NOFOLLOW); | ||
45 | function isfile_casesensitive(path) | ||
46 | isfile(path) || return false | ||
47 | path_basename = String(basename(path)) | ||
48 | local casepreserved_basename | ||
49 | header_size = 12 | ||
50 | buf = Vector{UInt8}(undef, length(path_basename) + header_size + 1) | ||
51 | while true | ||
52 | ret = ccall(:getattrlist, Cint, | ||
53 | (Cstring, Ptr{Cvoid}, Ptr{Cvoid}, Csize_t, Culong), | ||
54 | path, attr_list, buf, sizeof(buf), FSOPT_NOFOLLOW) | ||
55 | systemerror(:getattrlist, ret ≠ 0) | ||
56 | filename_length = GC.@preserve buf unsafe_load( | ||
57 | convert(Ptr{UInt32}, pointer(buf) + 8)) | ||
58 | if (filename_length + header_size) > length(buf) | ||
59 | resize!(buf, filename_length + header_size) | ||
60 | continue | ||
61 | end | ||
62 | casepreserved_basename = | ||
63 | view(buf, (header_size+1):(header_size+filename_length-1)) | ||
64 | break | ||
65 | end | ||
66 | # Hack to compensate for inability to create a string from a subarray with no allocations. | ||
67 | codeunits(path_basename) == casepreserved_basename && return true | ||
68 | |||
69 | # If there is no match, it's possible that the file does exist but HFS+ | ||
70 | # performed unicode normalization. See https://developer.apple.com/library/mac/qa/qa1235/_index.html. | ||
71 | isascii(path_basename) && return false | ||
72 | codeunits(Unicode.normalize(path_basename, :NFD)) == casepreserved_basename | ||
73 | end | ||
74 | else | ||
75 | # Generic fallback that performs a slow directory listing. | ||
76 | function isfile_casesensitive(path) | ||
77 | isfile(path) || return false | ||
78 | dir, filename = splitdir(path) | ||
79 | any(readdir(dir) .== filename) | ||
80 | end | ||
81 | end | ||
82 | |||
83 | ## SHA1 ## | ||
84 | |||
85 | struct SHA1 | ||
86 | bytes::Vector{UInt8} | ||
87 | function SHA1(bytes::Vector{UInt8}) | ||
88 | length(bytes) == 20 || | ||
89 | throw(ArgumentError("wrong number of bytes for SHA1 hash: $(length(bytes))")) | ||
90 | return new(bytes) | ||
91 | end | ||
92 | end | ||
93 | SHA1(s::AbstractString) = SHA1(hex2bytes(s)) | ||
94 | |||
95 | string(hash::SHA1) = bytes2hex(hash.bytes) | ||
96 | print(io::IO, hash::SHA1) = bytes2hex(io, hash.bytes) | ||
97 | show(io::IO, hash::SHA1) = print(io, "SHA1(\"", hash, "\")") | ||
98 | |||
99 | isless(a::SHA1, b::SHA1) = lexless(a.bytes, b.bytes) | ||
100 | hash(a::SHA1, h::UInt) = hash((SHA1, a.bytes), h) | ||
101 | ==(a::SHA1, b::SHA1) = a.bytes == b.bytes | ||
102 | |||
103 | # fake uuid5 function (for self-assigned UUIDs) | ||
104 | # TODO: delete and use real uuid5 once it's in stdlib | ||
105 | |||
106 | function uuid5(namespace::UUID, key::String) | ||
107 | u::UInt128 = 0 | ||
108 | h = hash(namespace) | ||
109 | for _ = 1:sizeof(u)÷sizeof(h) | ||
110 | u <<= sizeof(h) << 3 | ||
111 | u |= (h = hash(key, h)) | ||
112 | end | ||
113 | u &= 0xffffffffffff0fff3fffffffffffffff | ||
114 | u |= 0x00000000000050008000000000000000 | ||
115 | return UUID(u) | ||
116 | end | ||
117 | |||
118 | const ns_dummy_uuid = UUID("fe0723d6-3a44-4c41-8065-ee0f42c8ceab") | ||
119 | |||
120 | dummy_uuid(project_file::String) = isfile_casesensitive(project_file) ? | ||
121 | uuid5(ns_dummy_uuid, realpath(project_file)) : nothing | ||
122 | |||
123 | ## package path slugs: turning UUID + SHA1 into a pair of 4-byte "slugs" ## | ||
124 | |||
125 | const slug_chars = String(['A':'Z'; 'a':'z'; '0':'9']) | ||
126 | |||
127 | function slug(x::UInt32, p::Int) | ||
128 | sprint(sizehint=p) do io | ||
129 | n = length(slug_chars) | ||
130 | for i = 1:p | ||
131 | x, d = divrem(x, n) | ||
132 | write(io, slug_chars[1+d]) | ||
133 | end | ||
134 | end | ||
135 | end | ||
136 | |||
137 | function package_slug(uuid::UUID, p::Int=5) | ||
138 | crc = _crc32c(uuid) | ||
139 | return slug(crc, p) | ||
140 | end | ||
141 | |||
142 | function version_slug(uuid::UUID, sha1::SHA1, p::Int=5) | ||
143 | crc = _crc32c(uuid) | ||
144 | crc = _crc32c(sha1.bytes, crc) | ||
145 | return slug(crc, p) | ||
146 | end | ||
147 | |||
148 | ## package identification: determine unique identity of package to be loaded ## | ||
149 | |||
150 | find_package(args...) = locate_package(identify_package(args...)) | ||
151 | |||
152 | struct PkgId | ||
153 | uuid::Union{UUID,Nothing} | ||
154 | name::String | ||
155 | |||
156 | PkgId(u::UUID, name::AbstractString) = new(UInt128(u) == 0 ? nothing : u, name) | ||
157 | PkgId(::Nothing, name::AbstractString) = new(nothing, name) | ||
158 | end | ||
159 | PkgId(name::AbstractString) = PkgId(nothing, name) | ||
160 | |||
161 | function PkgId(m::Module, name::String = String(nameof(moduleroot(m)))) | ||
162 | uuid = UUID(ccall(:jl_module_uuid, NTuple{2, UInt64}, (Any,), m)) | ||
163 | UInt128(uuid) == 0 ? PkgId(name) : PkgId(uuid, name) | ||
164 | end | ||
165 | |||
166 | ==(a::PkgId, b::PkgId) = a.uuid == b.uuid && a.name == b.name | ||
167 | |||
168 | function hash(pkg::PkgId, h::UInt) | ||
169 | h += 0xc9f248583a0ca36c % UInt | ||
170 | h = hash(pkg.uuid, h) | ||
171 | h = hash(pkg.name, h) | ||
172 | return h | ||
173 | end | ||
174 | |||
175 | show(io::IO, pkg::PkgId) = | ||
176 | print(io, pkg.name, " [", pkg.uuid === nothing ? "top-level" : pkg.uuid, "]") | ||
177 | |||
178 | function binpack(pkg::PkgId) | ||
179 | io = IOBuffer() | ||
180 | write(io, UInt8(0)) | ||
181 | uuid = pkg.uuid | ||
182 | write(io, uuid === nothing ? UInt128(0) : UInt128(uuid)) | ||
183 | write(io, pkg.name) | ||
184 | return String(take!(io)) | ||
185 | end | ||
186 | |||
187 | function binunpack(s::String) | ||
188 | io = IOBuffer(s) | ||
189 | @assert read(io, UInt8) === 0x00 | ||
190 | uuid = read(io, UInt128) | ||
191 | name = read(io, String) | ||
192 | return PkgId(UUID(uuid), name) | ||
193 | end | ||
194 | |||
195 | function identify_package(where::Module, name::String)::Union{Nothing,PkgId} | ||
196 | identify_package(PkgId(where), name) | ||
197 | end | ||
198 | |||
199 | function identify_package(where::PkgId, name::String)::Union{Nothing,PkgId} | ||
200 | where.name === name && return where | ||
201 | where.uuid === nothing && return identify_package(name) | ||
202 | for env in load_path() | ||
203 | found_or_uuid = manifest_deps_get(env, where, name) | ||
204 | found_or_uuid isa UUID && return PkgId(found_or_uuid, name) | ||
205 | found_or_uuid && return nothing | ||
206 | end | ||
207 | return nothing | ||
208 | end | ||
209 | |||
210 | function identify_package(name::String)::Union{Nothing,PkgId} | ||
211 | for env in load_path() | ||
212 | found_or_uuid = project_deps_get(env, name) | ||
213 | found_or_uuid isa UUID && return PkgId(found_or_uuid, name) | ||
214 | found_or_uuid && return PkgId(name) | ||
215 | end | ||
216 | return nothing | ||
217 | end | ||
218 | |||
219 | function identify_package(name::String, names::String...) | ||
220 | pkg = identify_package(name) | ||
221 | pkg === nothing ? nothing : | ||
222 | pkg.uuid === nothing ? identify_package(names...) : | ||
223 | identify_package(pkg, names...) | ||
224 | end | ||
225 | |||
226 | function identify_package(where::PkgId, name::String, names::String...) | ||
227 | pkg = identify_package(where, name) | ||
228 | pkg === nothing ? nothing : | ||
229 | pkg.uuid === nothing ? identify_package(names...) : | ||
230 | identify_package(pkg, names...) | ||
231 | end | ||
232 | |||
233 | ## package location: given a package identity find file to load ## | ||
234 | |||
235 | function locate_package(pkg::PkgId)::Union{Nothing,String} | ||
236 | if pkg.uuid === nothing | ||
237 | for env in load_path() | ||
238 | found_or_uuid = project_deps_get(env, pkg.name) | ||
239 | found_or_uuid isa UUID && | ||
240 | return locate_package(PkgId(found_or_uuid, pkg.name)) | ||
241 | found_or_uuid && return implicit_manifest_uuid_path(env, pkg) | ||
242 | end | ||
243 | else | ||
244 | for env in load_path() | ||
245 | path = manifest_uuid_path(env, pkg) | ||
246 | path != nothing && return entry_path(path, pkg.name) | ||
247 | end | ||
248 | end | ||
249 | end | ||
250 | locate_package(::Nothing) = nothing | ||
251 | |||
252 | """ | ||
253 | pathof(m::Module) | ||
254 | |||
255 | Return the path of `m.jl` file that was used to `import` module `m`, | ||
256 | or `nothing` if `m` was not imported from a package. | ||
257 | |||
258 | Use [`dirname`](@ref) to get the directory part and [`basename`](@ref) | ||
259 | to get the file name part of the path. | ||
260 | """ | ||
261 | function pathof(m::Module) | ||
262 | pkgid = get(Base.module_keys, m, nothing) | ||
263 | pkgid === nothing && return nothing | ||
264 | return Base.locate_package(pkgid) | ||
265 | end | ||
266 | |||
267 | ## generic project & manifest API ## | ||
268 | |||
269 | const project_names = ("JuliaProject.toml", "Project.toml") | ||
270 | const manifest_names = ("JuliaManifest.toml", "Manifest.toml") | ||
271 | |||
272 | # return means | ||
273 | # - `false`: nothing to see here | ||
274 | # - `true`: `env` is an implicit environment | ||
275 | # - `path`: the path of an explicit project file | ||
276 | function env_project_file(env::String)::Union{Bool,String} | ||
277 | if isdir(env) | ||
278 | for proj in project_names | ||
279 | project_file = joinpath(env, proj) | ||
280 | isfile_casesensitive(project_file) && return project_file | ||
281 | end | ||
282 | return true | ||
283 | elseif basename(env) in project_names && isfile_casesensitive(env) | ||
284 | return env | ||
285 | end | ||
286 | return false | ||
287 | end | ||
288 | |||
289 | function project_deps_get(env::String, name::String)::Union{Bool,UUID} | ||
290 | project_file = env_project_file(env) | ||
291 | if project_file isa String | ||
292 | return explicit_project_deps_get(project_file, name) | ||
293 | end | ||
294 | project_file && implicit_project_deps_get(env, name) | ||
295 | end | ||
296 | |||
297 | function manifest_deps_get(env::String, where::PkgId, name::String)::Union{Bool,UUID} | ||
298 | @assert where.uuid !== nothing | ||
299 | project_file = env_project_file(env) | ||
300 | if project_file isa String | ||
301 | proj_name, proj_uuid = project_file_name_uuid_path(project_file, where.name) | ||
302 | if proj_name == where.name && proj_uuid == where.uuid | ||
303 | # `where` matches the project, use deps as manifest | ||
304 | found_or_uuid = explicit_project_deps_get(project_file, name) | ||
305 | return found_or_uuid isa UUID ? found_or_uuid : true | ||
306 | end | ||
307 | # look for `where` stanza in manifest file | ||
308 | manifest_file = project_file_manifest_path(project_file) | ||
309 | if isfile_casesensitive(manifest_file) | ||
310 | return explicit_manifest_deps_get(manifest_file, where.uuid, name) | ||
311 | end | ||
312 | return false # `where` stanza not found | ||
313 | end | ||
314 | project_file && implicit_manifest_deps_get(env, where, name) | ||
315 | end | ||
316 | |||
317 | function manifest_uuid_path(env::String, pkg::PkgId)::Union{Nothing,String} | ||
318 | project_file = env_project_file(env) | ||
319 | if project_file isa String | ||
320 | proj_name, proj_uuid, path = project_file_name_uuid_path(project_file, pkg.name) | ||
321 | proj_name == pkg.name && proj_uuid == pkg.uuid && return path | ||
322 | manifest_file = project_file_manifest_path(project_file) | ||
323 | if isfile_casesensitive(manifest_file) | ||
324 | return explicit_manifest_uuid_path(manifest_file, pkg) | ||
325 | end | ||
326 | return nothing | ||
327 | end | ||
328 | project_file ? implicit_manifest_uuid_path(env, pkg) : nothing | ||
329 | end | ||
330 | |||
331 | # regular expressions for scanning project & manifest files | ||
332 | |||
333 | const re_section = r"^\s*\[" | ||
334 | const re_array_of_tables = r"^\s*\[\s*\[" | ||
335 | const re_section_deps = r"^\s*\[\s*\"?deps\"?\s*\]\s*(?:#|$)" | ||
336 | const re_section_capture = r"^\s*\[\s*\[\s*\"?(\w+)\"?\s*\]\s*\]\s*(?:#|$)" | ||
337 | const re_subsection_deps = r"^\s*\[\s*\"?(\w+)\"?\s*\.\s*\"?deps\"?\s*\]\s*(?:#|$)" | ||
338 | const re_key_to_string = r"^\s*(\w+)\s*=\s*\"(.*)\"\s*(?:#|$)" | ||
339 | const re_uuid_to_string = r"^\s*uuid\s*=\s*\"(.*)\"\s*(?:#|$)" | ||
340 | const re_name_to_string = r"^\s*name\s*=\s*\"(.*)\"\s*(?:#|$)" | ||
341 | const re_path_to_string = r"^\s*path\s*=\s*\"(.*)\"\s*(?:#|$)" | ||
342 | const re_hash_to_string = r"^\s*git-tree-sha1\s*=\s*\"(.*)\"\s*(?:#|$)" | ||
343 | const re_manifest_to_string = r"^\s*manifest\s*=\s*\"(.*)\"\s*(?:#|$)" | ||
344 | const re_deps_to_any = r"^\s*deps\s*=\s*(.*?)\s*(?:#|$)" | ||
345 | |||
346 | # find project file's top-level UUID entry (or nothing) | ||
347 | function project_file_name_uuid_path(project_file::String, | ||
348 | name::String)::Tuple{String,UUID,String} | ||
349 | open(project_file) do io | ||
350 | uuid = dummy_uuid(project_file) | ||
351 | path = joinpath("src", "$name.jl") | ||
352 | for line in eachline(io) | ||
353 | occursin(re_section, line) && break | ||
354 | if (m = match(re_name_to_string, line)) != nothing | ||
355 | name = String(m.captures[1]) | ||
356 | elseif (m = match(re_uuid_to_string, line)) != nothing | ||
357 | uuid = UUID(m.captures[1]) | ||
358 | elseif (m = match(re_path_to_string, line)) != nothing | ||
359 | path = String(m.captures[1]) | ||
360 | end | ||
361 | end | ||
362 | path = joinpath(dirname(project_file), path) | ||
363 | return name, uuid, path | ||
364 | end | ||
365 | end | ||
366 | |||
367 | # find project file's corresponding manifest file | ||
368 | function project_file_manifest_path(project_file::String)::Union{Nothing,String} | ||
369 | open(project_file) do io | ||
370 | dir = abspath(dirname(project_file)) | ||
371 | for line in eachline(io) | ||
372 | occursin(re_section, line) && break | ||
373 | if (m = match(re_manifest_to_string, line)) != nothing | ||
374 | return normpath(joinpath(dir, m.captures[1])) | ||
375 | end | ||
376 | end | ||
377 | local manifest_file | ||
378 | for mfst in manifest_names | ||
379 | manifest_file = joinpath(dir, mfst) | ||
380 | isfile_casesensitive(manifest_file) && return manifest_file | ||
381 | end | ||
382 | return manifest_file | ||
383 | end | ||
384 | end | ||
385 | |||
386 | # find `name` in a manifest file and return its UUID | ||
387 | function manifest_file_name_uuid(manifest_file::String, name::String, io::IO)::Union{Nothing,UUID} | ||
388 | uuid = name′ = nothing | ||
389 | for line in eachline(io) | ||
390 | if (m = match(re_section_capture, line)) != nothing | ||
391 | name′ == name && break | ||
392 | name′ = String(m.captures[1]) | ||
393 | elseif (m = match(re_uuid_to_string, line)) != nothing | ||
394 | uuid = UUID(m.captures[1]) | ||
395 | end | ||
396 | end | ||
397 | name′ == name ? uuid : nothing | ||
398 | end | ||
399 | |||
400 | # given package dir and name, find an entry point | ||
401 | # and project file if one exists (or nothing if not) | ||
402 | function entry_point_and_project_file(dir::String, name::String)::Union{Tuple{Nothing,Nothing},Tuple{String,Nothing},Tuple{String,String}} | ||
403 | for entry in ("", joinpath(name, "src"), joinpath("$name.jl", "src")) | ||
404 | path = normpath(joinpath(dir, entry, "$name.jl")) | ||
405 | isfile_casesensitive(path) || continue | ||
406 | if !isempty(entry) | ||
407 | for proj in project_names | ||
408 | project_file = normpath(joinpath(dir, dirname(entry), proj)) | ||
409 | isfile_casesensitive(project_file) || continue | ||
410 | return path, project_file | ||
411 | end | ||
412 | end | ||
413 | return path, nothing | ||
414 | end | ||
415 | return nothing, nothing | ||
416 | end | ||
417 | |||
418 | # given a path and a name, return the entry point | ||
419 | function entry_path(path::String, name::String)::Union{Nothing,String} | ||
420 | isfile_casesensitive(path) && return normpath(path) | ||
421 | path = normpath(joinpath(path, "src", "$name.jl")) | ||
422 | isfile_casesensitive(path) ? path : nothing | ||
423 | end | ||
424 | entry_path(::Nothing, name::String) = nothing | ||
425 | |||
426 | # given a project path (project directory or entry point) | ||
427 | # return the project file | ||
428 | function package_path_to_project_file(path::String)::Union{Nothing,String} | ||
429 | if !isdir(path) | ||
430 | dir = dirname(path) | ||
431 | basename(dir) == "src" || return nothing | ||
432 | path = dirname(dir) | ||
433 | end | ||
434 | for proj in project_names | ||
435 | project_file = joinpath(path, proj) | ||
436 | isfile_casesensitive(project_file) && return project_file | ||
437 | end | ||
438 | end | ||
439 | |||
440 | ## explicit project & manifest API ## | ||
441 | |||
442 | # find project file root or deps `name => uuid` mapping | ||
443 | # - `false` means: did not find `name` | ||
444 | # - `true` means: found `name` without UUID (can't happen in explicit projects) | ||
445 | # - `uuid` means: found `name` with `uuid` in project file | ||
446 | |||
447 | function explicit_project_deps_get(project_file::String, name::String)::Union{Bool,UUID} | ||
448 | open(project_file) do io | ||
449 | root_name = nothing | ||
450 | root_uuid = dummy_uuid(project_file) | ||
451 | state = :top | ||
452 | for line in eachline(io) | ||
453 | if state == :top | ||
454 | if occursin(re_section, line) | ||
455 | root_name == name && return root_uuid | ||
456 | state = occursin(re_section_deps, line) ? :deps : :other | ||
457 | elseif (m = match(re_name_to_string, line)) != nothing | ||
458 | root_name = String(m.captures[1]) | ||
459 | elseif (m = match(re_uuid_to_string, line)) != nothing | ||
460 | root_uuid = UUID(m.captures[1]) | ||
461 | end | ||
462 | elseif state == :deps | ||
463 | if (m = match(re_key_to_string, line)) != nothing | ||
464 | m.captures[1] == name && return UUID(m.captures[2]) | ||
465 | end | ||
466 | end | ||
467 | if occursin(re_section, line) | ||
468 | state = occursin(re_section_deps, line) ? :deps : :other | ||
469 | end | ||
470 | end | ||
471 | return root_name == name && root_uuid | ||
472 | end | ||
473 | end | ||
474 | |||
475 | # find `where` stanza and `name` in its deps and return its UUID | ||
476 | # - `false` means: did not find `where` | ||
477 | # - `true` means: found `where` but `name` not in its deps | ||
478 | # - `uuid` means: found `where` and `name` mapped to `uuid` in its deps | ||
479 | |||
480 | function explicit_manifest_deps_get(manifest_file::String, where::UUID, name::String)::Union{Bool,UUID} | ||
481 | open(manifest_file) do io | ||
482 | uuid = deps = nothing | ||
483 | state = :other | ||
484 | for line in eachline(io) | ||
485 | if occursin(re_array_of_tables, line) | ||
486 | uuid == where && break | ||
487 | uuid = deps = nothing | ||
488 | state = :stanza | ||
489 | elseif state == :stanza | ||
490 | if (m = match(re_uuid_to_string, line)) != nothing | ||
491 | uuid = UUID(m.captures[1]) | ||
492 | elseif (m = match(re_deps_to_any, line)) != nothing | ||
493 | deps = String(m.captures[1]) | ||
494 | elseif occursin(re_subsection_deps, line) | ||
495 | state = :deps | ||
496 | elseif occursin(re_section, line) | ||
497 | state = :other | ||
498 | end | ||
499 | elseif state == :deps && uuid == where | ||
500 | if (m = match(re_key_to_string, line)) != nothing | ||
501 | m.captures[1] == name && return UUID(m.captures[2]) | ||
502 | end | ||
503 | end | ||
504 | end | ||
505 | uuid == where || return false | ||
506 | deps === nothing && return true | ||
507 | # TODO: handle inline table syntax | ||
508 | if deps[1] != '[' || deps[end] != ']' | ||
509 | @warn "Unexpected TOML deps format:\n$deps" | ||
510 | return nothing | ||
511 | end | ||
512 | occursin(repr(name), deps) || return true | ||
513 | seekstart(io) # rewind IO handle | ||
514 | return manifest_file_name_uuid(manifest_file, name, io) | ||
515 | end | ||
516 | end | ||
517 | |||
518 | # find `uuid` stanza, return the corresponding path | ||
519 | function explicit_manifest_uuid_path(manifest_file::String, pkg::PkgId)::Union{Nothing,String} | ||
520 | open(manifest_file) do io | ||
521 | uuid = name = path = hash = nothing | ||
522 | for line in eachline(io) | ||
523 | if (m = match(re_section_capture, line)) != nothing | ||
524 | uuid == pkg.uuid && break | ||
525 | name = String(m.captures[1]) | ||
526 | path = hash = nothing | ||
527 | elseif (m = match(re_uuid_to_string, line)) != nothing | ||
528 | uuid = UUID(m.captures[1]) | ||
529 | elseif (m = match(re_path_to_string, line)) != nothing | ||
530 | path = String(m.captures[1]) | ||
531 | elseif (m = match(re_hash_to_string, line)) != nothing | ||
532 | hash = SHA1(m.captures[1]) | ||
533 | end | ||
534 | end | ||
535 | uuid == pkg.uuid || return nothing | ||
536 | name == pkg.name || return nothing # TODO: allow a mismatch? | ||
537 | if path != nothing | ||
538 | path = normpath(abspath(dirname(manifest_file), path)) | ||
539 | return entry_path(path, name) | ||
540 | end | ||
541 | hash == nothing && return nothing | ||
542 | # Keep the 4 since it used to be the default | ||
543 | for slug in (version_slug(uuid, hash, 4), version_slug(uuid, hash)) | ||
544 | for depot in DEPOT_PATH | ||
545 | path = abspath(depot, "packages", name, slug) | ||
546 | ispath(path) && return entry_path(path, name) | ||
547 | end | ||
548 | end | ||
549 | end | ||
550 | end | ||
551 | |||
552 | ## implicit project & manifest API ## | ||
553 | |||
554 | # look for an entry point for `name`: | ||
555 | # - `false` means: did not find `name` | ||
556 | # - `true` means: found `name` without project file | ||
557 | # - `uuid` means: found `name` with project file with real or dummy `uuid` | ||
558 | function implicit_project_deps_get(dir::String, name::String)::Union{Bool,UUID} | ||
559 | path, project_file = entry_point_and_project_file(dir, name) | ||
560 | project_file == nothing && return path != nothing | ||
561 | proj_name, proj_uuid = project_file_name_uuid_path(project_file, name) | ||
562 | proj_name == name && proj_uuid | ||
563 | end | ||
564 | |||
565 | # look for an entry-point for `where` by name, check that UUID matches | ||
566 | # if there's a project file, look up `name` in its deps and return that | ||
567 | # - `false` means: did not find `where` | ||
568 | # - `true` means: found `where` but `name` not in its deps | ||
569 | # - `uuid` means: found `where` and `name` mapped to `uuid` in its deps | ||
570 | function implicit_manifest_deps_get(dir::String, where::PkgId, name::String)::Union{Bool,UUID} | ||
571 | @assert where.uuid !== nothing | ||
572 | project_file = entry_point_and_project_file(dir, where.name)[2] | ||
573 | project_file === nothing && return false | ||
574 | proj_name, proj_uuid = project_file_name_uuid_path(project_file, where.name) | ||
575 | proj_name == where.name && proj_uuid == where.uuid || return false | ||
576 | found_or_uuid = explicit_project_deps_get(project_file, name) | ||
577 | found_or_uuid isa UUID ? found_or_uuid : true | ||
578 | end | ||
579 | |||
580 | # look for an entry-point for `pkg` and return its path if UUID matches | ||
581 | function implicit_manifest_uuid_path(dir::String, pkg::PkgId)::Union{Nothing,String} | ||
582 | path, project_file = entry_point_and_project_file(dir, pkg.name) | ||
583 | pkg.uuid === nothing && project_file === nothing && return path | ||
584 | pkg.uuid === nothing || project_file === nothing && return nothing | ||
585 | proj_name, proj_uuid = project_file_name_uuid_path(project_file, pkg.name) | ||
586 | proj_name == pkg.name && proj_uuid == pkg.uuid ? path : nothing | ||
587 | end | ||
588 | |||
589 | ## other code loading functionality ## | ||
590 | |||
591 | function find_source_file(path::AbstractString) | ||
592 | (isabspath(path) || isfile(path)) && return path | ||
593 | base_path = joinpath(Sys.BINDIR::String, DATAROOTDIR, "julia", "base", path) | ||
594 | return isfile(base_path) ? base_path : nothing | ||
595 | end | ||
596 | |||
597 | cache_file_entry(pkg::PkgId) = joinpath( | ||
598 | "compiled", | ||
599 | "v$(VERSION.major).$(VERSION.minor)", | ||
600 | pkg.uuid === nothing ? "$(pkg.name).ji" : joinpath(pkg.name, "$(package_slug(pkg.uuid)).ji") | ||
601 | ) | ||
602 | |||
603 | function find_all_in_cache_path(pkg::PkgId) | ||
604 | paths = String[] | ||
605 | entry = cache_file_entry(pkg) | ||
606 | for depot in DEPOT_PATH | ||
607 | path = joinpath(depot, entry) | ||
608 | isfile_casesensitive(path) && push!(paths, path) | ||
609 | end | ||
610 | return paths | ||
611 | end | ||
612 | |||
613 | # these return either the array of modules loaded from the path / content given | ||
614 | # or an Exception that describes why it couldn't be loaded | ||
615 | # and it reconnects the Base.Docs.META | ||
616 | function _include_from_serialized(path::String, depmods::Vector{Any}) | ||
617 | sv = ccall(:jl_restore_incremental, Any, (Cstring, Any), path, depmods) | ||
618 | if isa(sv, Exception) | ||
619 | return sv | ||
620 | end | ||
621 | restored = sv[1] | ||
622 | if !isa(restored, Exception) | ||
623 | for M in restored::Vector{Any} | ||
624 | M = M::Module | ||
625 | if isdefined(M, Base.Docs.META) | ||
626 | push!(Base.Docs.modules, M) | ||
627 | end | ||
628 | if parentmodule(M) === M | ||
629 | register_root_module(M) | ||
630 | end | ||
631 | end | ||
632 | end | ||
633 | isassigned(sv, 2) && ccall(:jl_init_restored_modules, Cvoid, (Any,), sv[2]) | ||
634 | return restored | ||
635 | end | ||
636 | |||
637 | function _tryrequire_from_serialized(modkey::PkgId, build_id::UInt64, modpath::Union{Nothing, String}) | ||
638 | if root_module_exists(modkey) | ||
639 | M = root_module(modkey) | ||
640 | if PkgId(M) == modkey && module_build_id(M) === build_id | ||
641 | return M | ||
642 | end | ||
643 | else | ||
644 | if modpath === nothing | ||
645 | modpath = locate_package(modkey) | ||
646 | modpath === nothing && return nothing | ||
647 | end | ||
648 | mod = _require_search_from_serialized(modkey, String(modpath)) | ||
649 | if !isa(mod, Bool) | ||
650 | for callback in package_callbacks | ||
651 | invokelatest(callback, modkey) | ||
652 | end | ||
653 | for M in mod::Vector{Any} | ||
654 | if PkgId(M) == modkey && module_build_id(M) === build_id | ||
655 | return M | ||
656 | end | ||
657 | end | ||
658 | end | ||
659 | end | ||
660 | return nothing | ||
661 | end | ||
662 | |||
663 | function _require_from_serialized(path::String) | ||
664 | # loads a precompile cache file, ignoring stale_cachfile tests | ||
665 | # load all of the dependent modules first | ||
666 | local depmodnames | ||
667 | io = open(path, "r") | ||
668 | try | ||
669 | isvalid_cache_header(io) || return ArgumentError("Invalid header in cache file $path.") | ||
670 | depmodnames = parse_cache_header(io)[3] | ||
671 | isvalid_file_crc(io) || return ArgumentError("Invalid checksum in cache file $path.") | ||
672 | finally | ||
673 | close(io) | ||
674 | end | ||
675 | ndeps = length(depmodnames) | ||
676 | depmods = Vector{Any}(undef, ndeps) | ||
677 | for i in 1:ndeps | ||
678 | modkey, build_id = depmodnames[i] | ||
679 | dep = _tryrequire_from_serialized(modkey, build_id, nothing) | ||
680 | dep === nothing && return ErrorException("Required dependency $modkey failed to load from a cache file.") | ||
681 | depmods[i] = dep::Module | ||
682 | end | ||
683 | # then load the file | ||
684 | return _include_from_serialized(path, depmods) | ||
685 | end | ||
686 | |||
687 | # returns `true` if require found a precompile cache for this sourcepath, but couldn't load it | ||
688 | # returns `false` if the module isn't known to be precompilable | ||
689 | # returns the set of modules restored if the cache load succeeded | ||
690 | function _require_search_from_serialized(pkg::PkgId, sourcepath::String) | ||
691 | paths = find_all_in_cache_path(pkg) | ||
692 | for path_to_try in paths::Vector{String} | ||
693 | staledeps = stale_cachefile(sourcepath, path_to_try) | ||
694 | if staledeps === true | ||
695 | continue | ||
696 | end | ||
697 | # finish loading module graph into staledeps | ||
698 | for i in 1:length(staledeps) | ||
699 | dep = staledeps[i] | ||
700 | dep isa Module && continue | ||
701 | modpath, modkey, build_id = dep::Tuple{String, PkgId, UInt64} | ||
702 | dep = _tryrequire_from_serialized(modkey, build_id, modpath) | ||
703 | if dep === nothing | ||
704 | @debug "Required dependency $modkey failed to load from cache file for $modpath." | ||
705 | staledeps = true | ||
706 | break | ||
707 | end | ||
708 | staledeps[i] = dep::Module | ||
709 | end | ||
710 | if staledeps === true | ||
711 | continue | ||
712 | end | ||
713 | restored = _include_from_serialized(path_to_try, staledeps) | ||
714 | if isa(restored, Exception) | ||
715 | @debug "Deserialization checks failed while attempting to load cache from $path_to_try" exception=restored | ||
716 | else | ||
717 | return restored | ||
718 | end | ||
719 | end | ||
720 | return !isempty(paths) | ||
721 | end | ||
722 | |||
723 | # to synchronize multiple tasks trying to import/using something | ||
724 | const package_locks = Dict{PkgId,Condition}() | ||
725 | |||
726 | # to notify downstream consumers that a module was successfully loaded | ||
727 | # Callbacks take the form (mod::Base.PkgId) -> nothing. | ||
728 | # WARNING: This is an experimental feature and might change later, without deprecation. | ||
729 | const package_callbacks = Any[] | ||
730 | # to notify downstream consumers that a file has been included into a particular module | ||
731 | # Callbacks take the form (mod::Module, filename::String) -> nothing | ||
732 | # WARNING: This is an experimental feature and might change later, without deprecation. | ||
733 | const include_callbacks = Any[] | ||
734 | |||
735 | # used to optionally track dependencies when requiring a module: | ||
736 | const _concrete_dependencies = Pair{PkgId,UInt64}[] # these dependency versions are "set in stone", and the process should try to avoid invalidating them | ||
737 | const _require_dependencies = Any[] # a list of (mod, path, mtime) tuples that are the file dependencies of the module currently being precompiled | ||
738 | const _track_dependencies = Ref(false) # set this to true to track the list of file dependencies | ||
739 | function _include_dependency(mod::Module, _path::AbstractString) | ||
740 | prev = source_path(nothing) | ||
741 | if prev === nothing | ||
742 | path = abspath(_path) | ||
743 | else | ||
744 | path = normpath(joinpath(dirname(prev), _path)) | ||
745 | end | ||
746 | if _track_dependencies[] | ||
747 | push!(_require_dependencies, (mod, path, mtime(path))) | ||
748 | end | ||
749 | return path, prev | ||
750 | end | ||
751 | |||
752 | """ | ||
753 | include_dependency(path::AbstractString) | ||
754 | |||
755 | In a module, declare that the file specified by `path` (relative or absolute) is a | ||
756 | dependency for precompilation; that is, the module will need to be recompiled if this file | ||
757 | changes. | ||
758 | |||
759 | This is only needed if your module depends on a file that is not used via `include`. It has | ||
760 | no effect outside of compilation. | ||
761 | """ | ||
762 | function include_dependency(path::AbstractString) | ||
763 | _include_dependency(Main, path) | ||
764 | return nothing | ||
765 | end | ||
766 | |||
767 | # we throw PrecompilableError when a module doesn't want to be precompiled | ||
768 | struct PrecompilableError <: Exception end | ||
769 | function show(io::IO, ex::PrecompilableError) | ||
770 | print(io, "Declaring __precompile__(false) is not allowed in files that are being precompiled.") | ||
771 | end | ||
772 | precompilableerror(ex::PrecompilableError) = true | ||
773 | precompilableerror(ex::WrappedException) = precompilableerror(ex.error) | ||
774 | precompilableerror(@nospecialize ex) = false | ||
775 | |||
776 | # Call __precompile__(false) at the top of a tile prevent it from being precompiled (false) | ||
777 | """ | ||
778 | __precompile__(isprecompilable::Bool) | ||
779 | |||
780 | Specify whether the file calling this function is precompilable, defaulting to `true`. | ||
781 | If a module or file is *not* safely precompilable, it should call `__precompile__(false)` in | ||
782 | order to throw an error if Julia attempts to precompile it. | ||
783 | """ | ||
784 | @noinline function __precompile__(isprecompilable::Bool=true) | ||
785 | if !isprecompilable && ccall(:jl_generating_output, Cint, ()) != 0 | ||
786 | throw(PrecompilableError()) | ||
787 | end | ||
788 | nothing | ||
789 | end | ||
790 | |||
791 | # require always works in Main scope and loads files from node 1 | ||
792 | const toplevel_load = Ref(true) | ||
793 | |||
794 | const full_warning_showed = Ref(false) | ||
795 | const modules_warned_for = Set{PkgId}() | ||
796 | |||
797 | """ | ||
798 | require(into::Module, module::Symbol) | ||
799 | |||
800 | This function is part of the implementation of `using` / `import`, if a module is not | ||
801 | already defined in `Main`. It can also be called directly to force reloading a module, | ||
802 | regardless of whether it has been loaded before (for example, when interactively developing | ||
803 | libraries). | ||
804 | |||
805 | Loads a source file, in the context of the `Main` module, on every active node, searching | ||
806 | standard locations for files. `require` is considered a top-level operation, so it sets the | ||
807 | current `include` path but does not use it to search for files (see help for `include`). | ||
808 | This function is typically used to load library code, and is implicitly called by `using` to | ||
809 | load packages. | ||
810 | |||
811 | When searching for files, `require` first looks for package code in the global array | ||
812 | `LOAD_PATH`. `require` is case-sensitive on all platforms, including those with | ||
813 | case-insensitive filesystems like macOS and Windows. | ||
814 | |||
815 | For more details regarding code loading, see the manual. | ||
816 | """ | ||
817 | function require(into::Module, mod::Symbol) | ||
818 | uuidkey = identify_package(into, String(mod)) | ||
819 | # Core.println("require($(PkgId(into)), $mod) -> $uuidkey") | ||
820 | if uuidkey === nothing | ||
821 | where = PkgId(into) | ||
822 | if where.uuid === nothing | ||
823 | throw(ArgumentError(""" | ||
824 | Package $mod not found in current path: | ||
825 | - Run `import Pkg; Pkg.add($(repr(String(mod))))` to install the $mod package. | ||
826 | """)) | ||
827 | else | ||
828 | s = """ | ||
829 | Package $(where.name) does not have $mod in its dependencies: | ||
830 | - If you have $(where.name) checked out for development and have | ||
831 | added $mod as a dependency but haven't updated your primary | ||
832 | environment's manifest file, try `Pkg.resolve()`. | ||
833 | - Otherwise you may need to report an issue with $(where.name)""" | ||
834 | |||
835 | uuidkey = identify_package(PkgId(string(into)), String(mod)) | ||
836 | uuidkey === nothing && throw(ArgumentError(s)) | ||
837 | |||
838 | # fall back to toplevel loading with a warning | ||
839 | if !(where in modules_warned_for) | ||
840 | @warn string( | ||
841 | full_warning_showed[] ? "" : s, "\n", | ||
842 | string("Loading $(mod) into $(where.name) from project dependency, ", | ||
843 | "future warnings for $(where.name) are suppressed.") | ||
844 | ) _module = nothing _file = nothing _group = nothing | ||
845 | push!(modules_warned_for, where) | ||
846 | end | ||
847 | full_warning_showed[] = true | ||
848 | end | ||
849 | end | ||
850 | if _track_dependencies[] | ||
851 | push!(_require_dependencies, (into, binpack(uuidkey), 0.0)) | ||
852 | end | ||
853 | return require(uuidkey) | ||
854 | end | ||
855 | |||
856 | function require(uuidkey::PkgId) | ||
857 | if !root_module_exists(uuidkey) | ||
858 | _require(uuidkey) | ||
859 | # After successfully loading, notify downstream consumers | ||
860 | for callback in package_callbacks | ||
861 | invokelatest(callback, uuidkey) | ||
862 | end | ||
863 | end | ||
864 | return root_module(uuidkey) | ||
865 | end | ||
866 | |||
867 | const loaded_modules = Dict{PkgId,Module}() | ||
868 | const module_keys = IdDict{Module,PkgId}() # the reverse | ||
869 | |||
870 | is_root_module(m::Module) = haskey(module_keys, m) | ||
871 | root_module_key(m::Module) = module_keys[m] | ||
872 | |||
873 | function register_root_module(m::Module) | ||
874 | key = PkgId(m, String(nameof(m))) | ||
875 | if haskey(loaded_modules, key) | ||
876 | oldm = loaded_modules[key] | ||
877 | if oldm !== m | ||
878 | @warn "Replacing module `$(key.name)`" | ||
879 | end | ||
880 | end | ||
881 | loaded_modules[key] = m | ||
882 | module_keys[m] = key | ||
883 | nothing | ||
884 | end | ||
885 | |||
886 | register_root_module(Core) | ||
887 | register_root_module(Base) | ||
888 | register_root_module(Main) | ||
889 | |||
890 | # This is used as the current module when loading top-level modules. | ||
891 | # It has the special behavior that modules evaluated in it get added | ||
892 | # to the loaded_modules table instead of getting bindings. | ||
893 | baremodule __toplevel__ | ||
894 | using Base | ||
895 | end | ||
896 | |||
897 | # get a top-level Module from the given key | ||
898 | root_module(key::PkgId) = loaded_modules[key] | ||
899 | root_module(where::Module, name::Symbol) = | ||
900 | root_module(identify_package(where, String(name))) | ||
901 | |||
902 | root_module_exists(key::PkgId) = haskey(loaded_modules, key) | ||
903 | loaded_modules_array() = collect(values(loaded_modules)) | ||
904 | |||
905 | function unreference_module(key::PkgId) | ||
906 | if haskey(loaded_modules, key) | ||
907 | m = pop!(loaded_modules, key) | ||
908 | # need to ensure all modules are GC rooted; will still be referenced | ||
909 | # in module_keys | ||
910 | end | ||
911 | end | ||
912 | |||
913 | function _require(pkg::PkgId) | ||
914 | # handle recursive calls to require | ||
915 | loading = get(package_locks, pkg, false) | ||
916 | if loading !== false | ||
917 | # load already in progress for this module | ||
918 | wait(loading) | ||
919 | return | ||
920 | end | ||
921 | package_locks[pkg] = Condition() | ||
922 | |||
923 | last = toplevel_load[] | ||
924 | try | ||
925 | toplevel_load[] = false | ||
926 | # perform the search operation to select the module file require intends to load | ||
927 | path = locate_package(pkg) | ||
928 | if path === nothing | ||
929 | throw(ArgumentError(""" | ||
930 | Package $pkg is required but does not seem to be installed: | ||
931 | - Run `Pkg.instantiate()` to install all recorded dependencies. | ||
932 | """)) | ||
933 | end | ||
934 | |||
935 | # attempt to load the module file via the precompile cache locations | ||
936 | if JLOptions().use_compiled_modules != 0 | ||
937 | m = _require_search_from_serialized(pkg, path) | ||
938 | if !isa(m, Bool) | ||
939 | return | ||
940 | end | ||
941 | end | ||
942 | |||
943 | # if the module being required was supposed to have a particular version | ||
944 | # but it was not handled by the precompile loader, complain | ||
945 | for (concrete_pkg, concrete_build_id) in _concrete_dependencies | ||
946 | if pkg == concrete_pkg | ||
947 | @warn """Module $(pkg.name) with build ID $concrete_build_id is missing from the cache. | ||
948 | This may mean $pkg does not support precompilation but is imported by a module that does.""" | ||
949 | if JLOptions().incremental != 0 | ||
950 | # during incremental precompilation, this should be fail-fast | ||
951 | throw(PrecompilableError()) | ||
952 | end | ||
953 | end | ||
954 | end | ||
955 | |||
956 | if JLOptions().use_compiled_modules != 0 | ||
957 | if (0 == ccall(:jl_generating_output, Cint, ())) || (JLOptions().incremental != 0) | ||
958 | # spawn off a new incremental pre-compile task for recursive `require` calls | ||
959 | # or if the require search declared it was pre-compiled before (and therefore is expected to still be pre-compilable) | ||
960 | cachefile = compilecache(pkg, path) | ||
961 | if isa(cachefile, Exception) | ||
962 | if !precompilableerror(cachefile) | ||
963 | @warn "The call to compilecache failed to create a usable precompiled cache file for $pkg" exception=m | ||
964 | end | ||
965 | # fall-through to loading the file locally | ||
966 | else | ||
967 | m = _require_from_serialized(cachefile) | ||
968 | if isa(m, Exception) | ||
969 | @warn "The call to compilecache failed to create a usable precompiled cache file for $pkg" exception=m | ||
970 | else | ||
971 | return | ||
972 | end | ||
973 | end | ||
974 | end | ||
975 | end | ||
976 | |||
977 | # just load the file normally via include | ||
978 | # for unknown dependencies | ||
979 | uuid = pkg.uuid | ||
980 | uuid = (uuid === nothing ? (UInt64(0), UInt64(0)) : convert(NTuple{2, UInt64}, uuid)) | ||
981 | old_uuid = ccall(:jl_module_uuid, NTuple{2, UInt64}, (Any,), __toplevel__) | ||
982 | if uuid !== old_uuid | ||
983 | ccall(:jl_set_module_uuid, Cvoid, (Any, NTuple{2, UInt64}), __toplevel__, uuid) | ||
984 | end | ||
985 | try | ||
986 | include_relative(__toplevel__, path) | ||
987 | return | ||
988 | finally | ||
989 | if uuid !== old_uuid | ||
990 | ccall(:jl_set_module_uuid, Cvoid, (Any, NTuple{2, UInt64}), __toplevel__, old_uuid) | ||
991 | end | ||
992 | end | ||
993 | finally | ||
994 | toplevel_load[] = last | ||
995 | loading = pop!(package_locks, pkg) | ||
996 | notify(loading, all=true) | ||
997 | end | ||
998 | nothing | ||
999 | end | ||
1000 | |||
1001 | # relative-path load | ||
1002 | |||
1003 | """ | ||
1004 | include_string(m::Module, code::AbstractString, filename::AbstractString="string") | ||
1005 | |||
1006 | Like `include`, except reads code from the given string rather than from a file. | ||
1007 | """ | ||
1008 | include_string(m::Module, txt::String, fname::String) = | ||
1009 | ccall(:jl_load_file_string, Any, (Ptr{UInt8}, Csize_t, Cstring, Any), | ||
1010 | txt, sizeof(txt), fname, m) | ||
1011 | |||
1012 | include_string(m::Module, txt::AbstractString, fname::AbstractString="string") = | ||
1013 | include_string(m, String(txt), String(fname)) | ||
1014 | |||
1015 | function source_path(default::Union{AbstractString,Nothing}="") | ||
1016 | t = current_task() | ||
1017 | while true | ||
1018 | s = t.storage | ||
1019 | if s !== nothing && haskey(s, :SOURCE_PATH) | ||
1020 | return s[:SOURCE_PATH] | ||
1021 | end | ||
1022 | if t === t.parent | ||
1023 | return default | ||
1024 | end | ||
1025 | t = t.parent | ||
1026 | end | ||
1027 | end | ||
1028 | |||
1029 | function source_dir() | ||
1030 | p = source_path(nothing) | ||
1031 | p === nothing ? pwd() : dirname(p) | ||
1032 | end | ||
1033 | |||
1034 | include_relative(mod::Module, path::AbstractString) = include_relative(mod, String(path)) | ||
1035 |
40 (95.24%) samples spent in include_relative
function include_relative(mod::Module, _path::String)
0 (ex.), 40 (100.00%) (incl.) when called from include line 29 |
||
1036 | path, prev = _include_dependency(mod, _path) | ||
1037 | for callback in include_callbacks # to preserve order, must come before Core.include | ||
1038 | invokelatest(callback, mod, path) | ||
1039 | end | ||
1040 | tls = task_local_storage() | ||
1041 | tls[:SOURCE_PATH] = path | ||
1042 | local result | ||
1043 | try | ||
1044 | 40 (95.24%) |
40 (100.00%)
samples spent calling
include
result = Core.include(mod, path)
|
|
1045 | finally | ||
1046 | if prev === nothing | ||
1047 | delete!(tls, :SOURCE_PATH) | ||
1048 | else | ||
1049 | tls[:SOURCE_PATH] = prev | ||
1050 | end | ||
1051 | end | ||
1052 | return result | ||
1053 | end | ||
1054 | |||
1055 | """ | ||
1056 | Base.include([m::Module,] path::AbstractString) | ||
1057 | |||
1058 | Evaluate the contents of the input source file in the global scope of module `m`. | ||
1059 | Every module (except those defined with `baremodule`) has its own 1-argument | ||
1060 | definition of `include`, which evaluates the file in that module. | ||
1061 | Returns the result of the last evaluated expression of the input file. During including, | ||
1062 | a task-local include path is set to the directory containing the file. Nested calls to | ||
1063 | `include` will search relative to that path. This function is typically used to load source | ||
1064 | interactively, or to combine files in packages that are broken into multiple source files. | ||
1065 | """ | ||
1066 | Base.include # defined in sysimg.jl | ||
1067 | |||
1068 | """ | ||
1069 | evalfile(path::AbstractString, args::Vector{String}=String[]) | ||
1070 | |||
1071 | Load the file using [`Base.include`](@ref), evaluate all expressions, | ||
1072 | and return the value of the last one. | ||
1073 | """ | ||
1074 | function evalfile(path::AbstractString, args::Vector{String}=String[]) | ||
1075 | return Core.eval(Module(:__anon__), | ||
1076 | Expr(:toplevel, | ||
1077 | :(const ARGS = $args), | ||
1078 | :(eval(x) = $(Expr(:core, :eval))(__anon__, x)), | ||
1079 | :(include(x) = $(Expr(:top, :include))(__anon__, x)), | ||
1080 | :(include($path)))) | ||
1081 | end | ||
1082 | evalfile(path::AbstractString, args::Vector) = evalfile(path, String[args...]) | ||
1083 | |||
1084 | function load_path_setup_code(load_path::Bool=true) | ||
1085 | code = """ | ||
1086 | append!(empty!(Base.DEPOT_PATH), $(repr(map(abspath, DEPOT_PATH)))) | ||
1087 | append!(empty!(Base.DL_LOAD_PATH), $(repr(map(abspath, DL_LOAD_PATH)))) | ||
1088 | """ | ||
1089 | if load_path | ||
1090 | load_path = map(abspath, Base.load_path()) | ||
1091 | path_sep = Sys.iswindows() ? ';' : ':' | ||
1092 | any(path -> path_sep in path, load_path) && | ||
1093 | error("LOAD_PATH entries cannot contain $(repr(path_sep))") | ||
1094 | code *= """ | ||
1095 | append!(empty!(Base.LOAD_PATH), $(repr(load_path))) | ||
1096 | ENV["JULIA_LOAD_PATH"] = $(repr(join(load_path, Sys.iswindows() ? ';' : ':'))) | ||
1097 | Base.HOME_PROJECT[] = Base.ACTIVE_PROJECT[] = nothing | ||
1098 | """ | ||
1099 | end | ||
1100 | return code | ||
1101 | end | ||
1102 | |||
1103 | function create_expr_cache(input::String, output::String, concrete_deps::typeof(_concrete_dependencies), uuid::Union{Nothing,UUID}) | ||
1104 | rm(output, force=true) # Remove file if it exists | ||
1105 | code_object = """ | ||
1106 | while !eof(stdin) | ||
1107 | code = readuntil(stdin, '\\0') | ||
1108 | eval(Meta.parse(code)) | ||
1109 | end | ||
1110 | """ | ||
1111 | io = open(pipeline(detach(`$(julia_cmd()) -O0 | ||
1112 | --output-ji $output --output-incremental=yes | ||
1113 | --startup-file=no --history-file=no --warn-overwrite=yes | ||
1114 | --color=$(have_color ? "yes" : "no") | ||
1115 | --eval $code_object`), stderr=stderr), | ||
1116 | "w", stdout) | ||
1117 | in = io.in | ||
1118 | try | ||
1119 | write(in, """ | ||
1120 | begin | ||
1121 | $(Base.load_path_setup_code()) | ||
1122 | Base._track_dependencies[] = true | ||
1123 | Base.empty!(Base._concrete_dependencies) | ||
1124 | """) | ||
1125 | for (pkg, build_id) in concrete_deps | ||
1126 | pkg_str = if pkg.uuid === nothing | ||
1127 | "Base.PkgId($(repr(pkg.name)))" | ||
1128 | else | ||
1129 | "Base.PkgId(Base.UUID(\"$(pkg.uuid)\"), $(repr(pkg.name)))" | ||
1130 | end | ||
1131 | write(in, "Base.push!(Base._concrete_dependencies, $pkg_str => $(repr(build_id)))\n") | ||
1132 | end | ||
1133 | write(io, "end\0") | ||
1134 | uuid_tuple = uuid === nothing ? (0, 0) : convert(NTuple{2, UInt64}, uuid) | ||
1135 | write(in, "ccall(:jl_set_module_uuid, Cvoid, (Any, NTuple{2, UInt64}), Base.__toplevel__, $uuid_tuple)\0") | ||
1136 | source = source_path(nothing) | ||
1137 | if source !== nothing | ||
1138 | write(in, "task_local_storage()[:SOURCE_PATH] = $(repr(source))\0") | ||
1139 | end | ||
1140 | write(in, """ | ||
1141 | try | ||
1142 | Base.include(Base.__toplevel__, $(repr(abspath(input)))) | ||
1143 | catch ex | ||
1144 | Base.precompilableerror(ex) || Base.rethrow(ex) | ||
1145 | Base.@debug "Aborting `createexprcache'" exception=(Base.ErrorException("Declaration of __precompile__(false) not allowed"), Base.catch_backtrace()) | ||
1146 | Base.exit(125) # we define status = 125 means PrecompileableError | ||
1147 | end\0""") | ||
1148 | # TODO: cleanup is probably unnecessary here | ||
1149 | if source !== nothing | ||
1150 | write(in, "delete!(task_local_storage(), :SOURCE_PATH)\0") | ||
1151 | end | ||
1152 | write(in, "ccall(:jl_set_module_uuid, Cvoid, (Any, NTuple{2, UInt64}), Base.__toplevel__, (0, 0))\0") | ||
1153 | close(in) | ||
1154 | catch ex | ||
1155 | close(in) | ||
1156 | process_running(io) && Timer(t -> kill(io), 5.0) # wait a short time before killing the process to give it a chance to clean up on its own first | ||
1157 | rethrow(ex) | ||
1158 | end | ||
1159 | return io | ||
1160 | end | ||
1161 | |||
1162 | """ | ||
1163 | Base.compilecache(module::PkgId) | ||
1164 | |||
1165 | Creates a precompiled cache file for a module and all of its dependencies. | ||
1166 | This can be used to reduce package load times. Cache files are stored in | ||
1167 | `DEPOT_PATH[1]/compiled`. See [Module initialization and precompilation](@ref) | ||
1168 | for important notes. | ||
1169 | """ | ||
1170 | function compilecache(pkg::PkgId) | ||
1171 | path = locate_package(pkg) | ||
1172 | path === nothing && throw(ArgumentError("$pkg not found during precompilation")) | ||
1173 | return compilecache(pkg, path) | ||
1174 | end | ||
1175 | function compilecache(pkg::PkgId, path::String) | ||
1176 | # decide where to put the resulting cache file | ||
1177 | cachefile = abspath(DEPOT_PATH[1], cache_file_entry(pkg)) | ||
1178 | cachepath = dirname(cachefile) | ||
1179 | isdir(cachepath) || mkpath(cachepath) | ||
1180 | # build up the list of modules that we want the precompile process to preserve | ||
1181 | concrete_deps = copy(_concrete_dependencies) | ||
1182 | for (key, mod) in loaded_modules | ||
1183 | if !(mod === Main || mod === Core || mod === Base) | ||
1184 | push!(concrete_deps, key => module_build_id(mod)) | ||
1185 | end | ||
1186 | end | ||
1187 | # run the expression and cache the result | ||
1188 | verbosity = isinteractive() ? CoreLogging.Info : CoreLogging.Debug | ||
1189 | if isfile(cachefile) | ||
1190 | @logmsg verbosity "Recompiling stale cache file $cachefile for $pkg" | ||
1191 | else | ||
1192 | @logmsg verbosity "Precompiling $pkg" | ||
1193 | end | ||
1194 | p = create_expr_cache(path, cachefile, concrete_deps, pkg.uuid) | ||
1195 | if success(p) | ||
1196 | # append checksum to the end of the .ji file: | ||
1197 | open(cachefile, "a+") do f | ||
1198 | write(f, _crc32c(seekstart(f))) | ||
1199 | end | ||
1200 | elseif p.exitcode == 125 | ||
1201 | return PrecompilableError() | ||
1202 | else | ||
1203 | error("Failed to precompile $pkg to $cachefile.") | ||
1204 | end | ||
1205 | return cachefile | ||
1206 | end | ||
1207 | |||
1208 | module_build_id(m::Module) = ccall(:jl_module_build_id, UInt64, (Any,), m) | ||
1209 | |||
1210 | isvalid_cache_header(f::IOStream) = (0 != ccall(:jl_read_verify_header, Cint, (Ptr{Cvoid},), f.ios)) | ||
1211 | isvalid_file_crc(f::IOStream) = (_crc32c(seekstart(f), filesize(f) - 4) == read(f, UInt32)) | ||
1212 | |||
1213 | function parse_cache_header(f::IO) | ||
1214 | modules = Vector{Pair{PkgId, UInt64}}() | ||
1215 | while true | ||
1216 | n = read(f, Int32) | ||
1217 | n == 0 && break | ||
1218 | sym = String(read(f, n)) # module name | ||
1219 | uuid = UUID((read(f, UInt64), read(f, UInt64))) # pkg UUID | ||
1220 | build_id = read(f, UInt64) # build UUID (mostly just a timestamp) | ||
1221 | push!(modules, PkgId(uuid, sym) => build_id) | ||
1222 | end | ||
1223 | totbytes = read(f, Int64) # total bytes for file dependencies | ||
1224 | # read the list of requirements | ||
1225 | # and split the list into include and requires statements | ||
1226 | includes = Tuple{PkgId, String, Float64}[] | ||
1227 | requires = Pair{PkgId, PkgId}[] | ||
1228 | while true | ||
1229 | n2 = read(f, Int32) | ||
1230 | n2 == 0 && break | ||
1231 | depname = String(read(f, n2)) | ||
1232 | mtime = read(f, Float64) | ||
1233 | n1 = read(f, Int32) | ||
1234 | # map ids to keys | ||
1235 | modkey = (n1 == 0) ? PkgId("") : modules[n1].first | ||
1236 | if n1 != 0 | ||
1237 | # consume (and ignore) the module path too | ||
1238 | while true | ||
1239 | n1 = read(f, Int32) | ||
1240 | totbytes -= 4 | ||
1241 | n1 == 0 && break | ||
1242 | skip(f, n1) # String(read(f, n1)) | ||
1243 | totbytes -= n1 | ||
1244 | end | ||
1245 | end | ||
1246 | if depname[1] == '\0' | ||
1247 | push!(requires, modkey => binunpack(depname)) | ||
1248 | else | ||
1249 | push!(includes, (modkey, depname, mtime)) | ||
1250 | end | ||
1251 | totbytes -= 4 + 4 + n2 + 8 | ||
1252 | end | ||
1253 | @assert totbytes == 12 "header of cache file appears to be corrupt" | ||
1254 | srctextpos = read(f, Int64) | ||
1255 | # read the list of modules that are required to be present during loading | ||
1256 | required_modules = Vector{Pair{PkgId, UInt64}}() | ||
1257 | while true | ||
1258 | n = read(f, Int32) | ||
1259 | n == 0 && break | ||
1260 | sym = String(read(f, n)) # module name | ||
1261 | uuid = UUID((read(f, UInt64), read(f, UInt64))) # pkg UUID | ||
1262 | build_id = read(f, UInt64) # build id | ||
1263 | push!(required_modules, PkgId(uuid, sym) => build_id) | ||
1264 | end | ||
1265 | return modules, (includes, requires), required_modules, srctextpos | ||
1266 | end | ||
1267 | |||
1268 | function parse_cache_header(cachefile::String) | ||
1269 | io = open(cachefile, "r") | ||
1270 | try | ||
1271 | !isvalid_cache_header(io) && throw(ArgumentError("Invalid header in cache file $cachefile.")) | ||
1272 | return parse_cache_header(io) | ||
1273 | finally | ||
1274 | close(io) | ||
1275 | end | ||
1276 | end | ||
1277 | |||
1278 | function cache_dependencies(f::IO) | ||
1279 | defs, (includes, requires), modules = parse_cache_header(f) | ||
1280 | return modules, map(mod_fl_mt -> (mod_fl_mt[2], mod_fl_mt[3]), includes) # discard the module | ||
1281 | end | ||
1282 | |||
1283 | function cache_dependencies(cachefile::String) | ||
1284 | io = open(cachefile, "r") | ||
1285 | try | ||
1286 | !isvalid_cache_header(io) && throw(ArgumentError("Invalid header in cache file $cachefile.")) | ||
1287 | return cache_dependencies(io) | ||
1288 | finally | ||
1289 | close(io) | ||
1290 | end | ||
1291 | end | ||
1292 | |||
1293 | function read_dependency_src(io::IO, filename::AbstractString) | ||
1294 | modules, (includes, requires), required_modules, srctextpos = parse_cache_header(io) | ||
1295 | srctextpos == 0 && error("no source-text stored in cache file") | ||
1296 | seek(io, srctextpos) | ||
1297 | return _read_dependency_src(io, filename) | ||
1298 | end | ||
1299 | |||
1300 | function _read_dependency_src(io::IO, filename::AbstractString) | ||
1301 | while !eof(io) | ||
1302 | filenamelen = read(io, Int32) | ||
1303 | filenamelen == 0 && break | ||
1304 | fn = String(read(io, filenamelen)) | ||
1305 | len = read(io, UInt64) | ||
1306 | if fn == filename | ||
1307 | return String(read(io, len)) | ||
1308 | end | ||
1309 | seek(io, position(io) + len) | ||
1310 | end | ||
1311 | error(filename, " is not stored in the source-text cache") | ||
1312 | end | ||
1313 | |||
1314 | function read_dependency_src(cachefile::String, filename::AbstractString) | ||
1315 | io = open(cachefile, "r") | ||
1316 | try | ||
1317 | !isvalid_cache_header(io) && throw(ArgumentError("Invalid header in cache file $cachefile.")) | ||
1318 | return read_dependency_src(io, filename) | ||
1319 | finally | ||
1320 | close(io) | ||
1321 | end | ||
1322 | end | ||
1323 | |||
1324 | # returns true if it "cachefile.ji" is stale relative to "modpath.jl" | ||
1325 | # otherwise returns the list of dependencies to also check | ||
1326 | function stale_cachefile(modpath::String, cachefile::String) | ||
1327 | io = open(cachefile, "r") | ||
1328 | try | ||
1329 | if !isvalid_cache_header(io) | ||
1330 | @debug "Rejecting cache file $cachefile due to it containing an invalid cache header" | ||
1331 | return true # invalid cache file | ||
1332 | end | ||
1333 | (modules, (includes, requires), required_modules) = parse_cache_header(io) | ||
1334 | modules = Dict{PkgId, UInt64}(modules) | ||
1335 | |||
1336 | # Check if transitive dependencies can be fulfilled | ||
1337 | ndeps = length(required_modules) | ||
1338 | depmods = Vector{Any}(undef, ndeps) | ||
1339 | for i in 1:ndeps | ||
1340 | req_key, req_build_id = required_modules[i] | ||
1341 | # Module is already loaded | ||
1342 | if root_module_exists(req_key) | ||
1343 | M = root_module(req_key) | ||
1344 | if PkgId(M) == req_key && module_build_id(M) === req_build_id | ||
1345 | depmods[i] = M | ||
1346 | else | ||
1347 | @debug "Rejecting cache file $cachefile because module $req_key is already loaded and incompatible." | ||
1348 | return true # Won't be able to fulfill dependency | ||
1349 | end | ||
1350 | else | ||
1351 | path = locate_package(req_key) | ||
1352 | if path === nothing | ||
1353 | @debug "Rejecting cache file $cachefile because dependency $req_key not found." | ||
1354 | return true # Won't be able to fulfill dependency | ||
1355 | end | ||
1356 | depmods[i] = (path, req_key, req_build_id) | ||
1357 | end | ||
1358 | end | ||
1359 | |||
1360 | # check if this file is going to provide one of our concrete dependencies | ||
1361 | # or if it provides a version that conflicts with our concrete dependencies | ||
1362 | # or neither | ||
1363 | skip_timecheck = false | ||
1364 | for (req_key, req_build_id) in _concrete_dependencies | ||
1365 | build_id = get(modules, req_key, UInt64(0)) | ||
1366 | if build_id !== UInt64(0) | ||
1367 | if build_id === req_build_id | ||
1368 | skip_timecheck = true | ||
1369 | break | ||
1370 | end | ||
1371 | @debug "Rejecting cache file $cachefile because it provides the wrong uuid (got $build_id) for $mod (want $req_build_id)" | ||
1372 | return true # cachefile doesn't provide the required version of the dependency | ||
1373 | end | ||
1374 | end | ||
1375 | |||
1376 | # now check if this file is fresh relative to its source files | ||
1377 | if !skip_timecheck | ||
1378 | if !samefile(includes[1][2], modpath) | ||
1379 | @debug "Rejecting cache file $cachefile because it is for file $(includes[1][2])) not file $modpath" | ||
1380 | return true # cache file was compiled from a different path | ||
1381 | end | ||
1382 | for (modkey, req_modkey) in requires | ||
1383 | # verify that `require(modkey, name(req_modkey))` ==> `req_modkey` | ||
1384 | if identify_package(modkey, req_modkey.name) != req_modkey | ||
1385 | @debug "Rejecting cache file $cachefile because uuid mapping for $modkey => $req_modkey has changed" | ||
1386 | return true | ||
1387 | end | ||
1388 | end | ||
1389 | for (_, f, ftime_req) in includes | ||
1390 | # Issue #13606: compensate for Docker images rounding mtimes | ||
1391 | # Issue #20837: compensate for GlusterFS truncating mtimes to microseconds | ||
1392 | ftime = mtime(f) | ||
1393 | if ftime != ftime_req && ftime != floor(ftime_req) && ftime != trunc(ftime_req, digits=6) | ||
1394 | @debug "Rejecting stale cache file $cachefile (mtime $ftime_req) because file $f (mtime $ftime) has changed" | ||
1395 | return true | ||
1396 | end | ||
1397 | end | ||
1398 | end | ||
1399 | |||
1400 | if !isvalid_file_crc(io) | ||
1401 | @debug "Rejecting cache file $cachefile because it has an invalid checksum" | ||
1402 | return true | ||
1403 | end | ||
1404 | |||
1405 | return depmods # fresh cachefile | ||
1406 | finally | ||
1407 | close(io) | ||
1408 | end | ||
1409 | end | ||
1410 | |||
1411 | """ | ||
1412 | @__FILE__ -> AbstractString | ||
1413 | |||
1414 | Expand to a string with the path to the file containing the | ||
1415 | macrocall, or an empty string if evaluated by `julia -e <expr>`. | ||
1416 | Return `nothing` if the macro was missing parser source information. | ||
1417 | Alternatively see [`PROGRAM_FILE`](@ref). | ||
1418 | """ | ||
1419 | macro __FILE__() | ||
1420 | __source__.file === nothing && return nothing | ||
1421 | return String(__source__.file) | ||
1422 | end | ||
1423 | |||
1424 | """ | ||
1425 | @__DIR__ -> AbstractString | ||
1426 | |||
1427 | Expand to a string with the absolute path to the directory of the file | ||
1428 | containing the macrocall. | ||
1429 | Return the current working directory if run from a REPL or if evaluated by `julia -e <expr>`. | ||
1430 | """ | ||
1431 | macro __DIR__() | ||
1432 | __source__.file === nothing && return nothing | ||
1433 | return abspath(dirname(String(__source__.file))) | ||
1434 | end |