Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 4 additions & 0 deletions projectgraph/cha.go
Original file line number Diff line number Diff line change
Expand Up @@ -66,14 +66,17 @@ func (b *builder) mro(class string) []string {
return order
}
b.mros[class] = []string{class}
b.ordering++
var order []string
if rec, ok := b.classes[class]; ok && profileOf(rec.ix.Language).c3 {
order = b.c3(class)
}
if order == nil {
order = b.depthFirst(class, 0, map[string]bool{})
}
b.ordering--
b.mros[class] = order
b.memo(func() { delete(b.mros, class) })
return order
}

Expand Down Expand Up @@ -181,6 +184,7 @@ func (b *builder) externalBase(class string) (string, bool) {
if !ok {
base, _ = b.findExternalBase(class)
b.externalBases[class] = base
b.memo(func() { delete(b.externalBases, class) })
}
return base, base != ""
}
Expand Down
2 changes: 1 addition & 1 deletion projectgraph/graph.go
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,7 @@ import (
// Version is the version of the link rules. Bump it when a rule that makes a
// node, an edge, a root or an unresolved site changes, so a diff of two
// reports shows the change.
const Version = "18"
const Version = "19"

type NodeKind string

Expand Down
37 changes: 34 additions & 3 deletions projectgraph/link.go
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,7 @@ import (
"io/fs"
"slices"
"strings"
"sync/atomic"
"time"

"github.com/safedep/code/fileindex"
Expand Down Expand Up @@ -78,13 +79,20 @@ func Build(ctx context.Context, indexes []*fileindex.Index, opts Options) (*Grap
if err != nil {
return nil, err
}
defer context.AfterFunc(ctx, func() { b.stopped.Store(true) })()
if err := b.linkAll(ctx); err != nil {
return nil, err
}
if err := b.markRoots(ctx); err != nil {
return nil, err
}
g := b.finish()
// A stop cuts the resolutions short, so the graph of a stopped build
// lacks edges and goes to no caller. finish resolves the external keys
// of the units, so the check comes after it.
if err := ctx.Err(); err != nil {
return nil, err
}
g.Stats.Duration = time.Since(start)
return g, nil
}
Expand Down Expand Up @@ -134,6 +142,20 @@ type builder struct {
// walk of a unit reads now for the result type of an external key.
reading map[string]bool
stepping map[nameKey]bool
// expanding holds the scopes whose wildcard imports a resolution reads
// now, with the modules of the last whole pass.
expanding map[*scope]*wildcardRead
// stopped marks a build whose context ended. The walk and the reads of
// wildcard imports then end at once. A context check only between files
// lets the resolutions of one file run on past the budget of the caller,
// and a cycle of glob uses can keep them running for hours.
stopped atomic.Bool
// wildcardMods holds the modules of the wildcard imports of each scope
// whose read no cycle guard cut, as wildcardModules describes. asked
// holds the Rust paths that the read whose modules can go to the table
// asks the resolving guard for, and is nil with no such read.
wildcardMods map[*scope]*wildcardMods
asked map[[2]string]bool
// unitTypes holds the type that a unit of dependency code gives for
// each external key, or "".
unitTypes map[string]string
Expand All @@ -157,8 +179,14 @@ type builder struct {
// files declare a package or a namespace.
namespaces map[langcode.Code]*namespaces

mros map[string][]string
children map[string][]string
mros map[string][]string
// ordering counts the classes whose MRO a lookup reads now. A read of
// such a class gets the class alone.
ordering int
// tentative holds an undo step for each memo write made while a read
// of wildcard imports is in progress, as wildcardModules describes.
tentative []func()
children map[string][]string
// aliases holds the definitions of each class that the language joins
// to one class, the class too.
aliases map[string][]string
Expand Down Expand Up @@ -274,6 +302,7 @@ func newTables(indexes []*fileindex.Index, opts Options) *builder {
defs: make(map[string]*fileindex.Definition, defs),
reading: map[string]bool{},
stepping: map[nameKey]bool{},
expanding: map[*scope]*wildcardRead{},
unitTypes: map[string]string{},
findingBases: map[string]bool{},
state: map[langcode.Code]any{},
Expand Down Expand Up @@ -346,10 +375,12 @@ func (b *builder) linkAll(ctx context.Context) error {
}
}
// The classes are whole from here, so the members of a group of
// aliases can be one table.
// aliases can be one table. A read of the wildcard imports of a scope
// before this point can miss a class, so wildcardMods starts here.
b.groupMembers = map[string]map[string]string{}
b.groupRecords = map[string][]classRecord{}
b.externalBases = map[string]string{}
b.wildcardMods = map[*scope]*wildcardMods{}

for _, p := range b.paths {
if err := ctx.Err(); err != nil {
Expand Down
2 changes: 2 additions & 0 deletions projectgraph/names.go
Original file line number Diff line number Diff line change
Expand Up @@ -29,6 +29,7 @@ func (b *builder) resolveName(module, name string, hops int) (string, *failure)
delete(b.visiting, k)
if fail == nil || !fail.cut {
b.names[k] = resolution{key: key, fail: fail}
b.memo(func() { delete(b.names, k) })
}
return key, fail
}
Expand Down Expand Up @@ -223,6 +224,7 @@ func (b *builder) bound(key string) (string, bool) {
return "", false
}
b.bindings[key] = target
b.memo(func() { delete(b.bindings, key) })
if b.g.Nodes[target] != nil {
b.link(sc.ix.Language, key, target, b.siteOf(key), EdgeExport, ruleAlias)
}
Expand Down
138 changes: 133 additions & 5 deletions projectgraph/resolve.go
Original file line number Diff line number Diff line change
Expand Up @@ -472,16 +472,15 @@ func (b *builder) resolveScope(sc *scope, caller, target string) []resolution {
}

// wildcardImported returns the definitions of the name in the packages
// that the file imports with a Go dot import or a Java wildcard import, and
// the members of the name in the classes of a Java static wildcard import.
// that the file imports with a Go dot import, a Java wildcard import or a
// Rust glob use, and the members of the name in the classes of a Java
// static wildcard import.
// A Java wildcard import of a package of the project names its class
// exactly, so its edge is direct.
func (b *builder) wildcardImported(sc *scope, name string) []string {
var keys []string
for _, imp := range sc.wildcards {
mod, _, ok := b.resolveModule(sc.ix, imp.Module)
for _, mod := range b.wildcardModules(sc) {
switch {
case !ok:
case b.units[mod] != nil:
keys = append(keys, b.units[mod].names[name]...)
case b.isType(mod):
Expand All @@ -493,6 +492,135 @@ func (b *builder) wildcardImported(sc *scope, name string) []string {
return keys
}

// wildcardRead is a read of the wildcard imports of a scope that a
// resolution makes now. mods holds the modules of the last whole pass, and
// read marks a pass whose resolutions read the same scope again.
type wildcardRead struct {
mods []string
read bool
}

// wildcardMods holds the modules of the wildcard imports of a scope, and
// the Rust paths that the read of them asked the resolving guard for.
type wildcardMods struct {
mods []string
asked map[[2]string]bool
}

// wildcardModules returns the modules of the wildcard imports of a scope,
// in the order of the imports. The resolution of a module can walk a unit
// whose name lookup reads the wildcard imports of the same scope, as the
// eight glob uses of src/sys.rs of time 0.1.45 do. If each such read
// resolved every import again, the work would grow with the orders of the
// imports, and the link step of a project with that file would not end.
//
// So a read of a scope that a resolution reads now gets the modules of the
// last whole pass, and the passes repeat until the modules stay the same.
// A module that only an import after it brings into scope thus resolves.
// A memo that a pass writes from the modules of the last pass can lack a
// module for the whole build. The MRO of a class whose base only the next
// pass brings into scope is such a memo, and a call of a method of the base
// then gets no edge. So a pass whose modules change undoes its memo writes.
//
// A cycle guard that an outer resolution holds can cut a read short.
// app/src/prelude.rs has pub use core_lib::*, and the resolution of that
// path reads the glob use again under its own resolving guard. A table
// that kept that list gave no module for the whole build, and a target
// under the crate read not-found. So the table keeps and gives a list only
// where uncut holds, and any other read resolves the imports again.
//
// A cycle of wildcard imports across files still costs time that grows
// with the number of files. A file that each other file of the cycle reads
// with a glob use of a name that no file defines is such a cycle.
func (b *builder) wildcardModules(sc *scope) []string {
if b.stopped.Load() {
return nil
}
if r := b.expanding[sc]; r != nil {
r.read = true
return r.mods
}
if w := b.wildcardMods[sc]; w != nil && b.uncut(w.asked) {
return w.mods
}
// A read inside another read adds the paths that it asks for to the
// record of the outer one.
outer := b.wildcardMods != nil && b.asked == nil
if outer {
b.asked = map[[2]string]bool{}
}
r := &wildcardRead{}
b.expanding[sc] = r
for range len(sc.wildcards) + 1 {
r.read = false
mark := len(b.tentative)
var mods []string
for _, imp := range sc.wildcards {
if mod, _, ok := b.resolveModule(sc.ix, imp.Module); ok {
mods = append(mods, mod)
}
}
same := slices.Equal(mods, r.mods)
r.mods = mods
if !r.read || same {
break
}
b.undo(mark)
}
delete(b.expanding, sc)
if len(b.expanding) == 0 {
clear(b.tentative)
b.tentative = b.tentative[:0]
}
if outer {
if b.uncut(b.asked) {
b.wildcardMods[sc] = &wildcardMods{mods: r.mods, asked: b.asked}
}
b.asked = nil
}
return r.mods
}

// memo logs the undo step of a memo write while a read of wildcard imports
// is in progress.
func (b *builder) memo(undo func()) {
if len(b.expanding) > 0 {
b.tentative = append(b.tentative, undo)
}
}

// undo reverts the memo writes after the mark, newest first.
func (b *builder) undo(mark int) {
for i := len(b.tentative) - 1; i >= mark; i-- {
b.tentative[i]()
}
clear(b.tentative[mark:])
b.tentative = b.tentative[:mark]
}

// uncut reports whether a read of the wildcard imports of a scope that
// asks the resolving guard for the paths gets no answer from a resolution
// in progress. Such a read gives the same modules each time. The guards
// are set and cleared in the order of a stack. So at the end of a read, a
// path in progress was set before the read, and the guard cut the read if
// the read asked for it.
//
// uncut compares the paths of the resolving guard only. On the Rust
// corpus, the reads under any other guard cost less than 3% of the read
// time. So any other state in progress fails the check.
func (b *builder) uncut(asked map[[2]string]bool) bool {
s := b.rustState()
if len(b.expanding)+len(b.visiting)+len(b.reading)+len(b.stepping)+len(b.findingBases)+len(s.routing)+b.ordering > 0 {
return false
}
for k := range s.resolving {
if asked[k] {
return false
}
}
return true
}

// superKeyword is the receiver of a call of a method of the base class, as
// super().load() in Python and super.load() in JavaScript. A bare call of it
// in JavaScript calls the constructor of the base class.
Expand Down
3 changes: 3 additions & 0 deletions projectgraph/rust.go
Original file line number Diff line number Diff line change
Expand Up @@ -448,6 +448,9 @@ func joinPath(module string, parts []string) string {
func (b *builder) resolveRust(importer, written string) (string, bool) {
s := b.rustState()
k := [2]string{importer, written}
if b.asked != nil {
b.asked[k] = true
}
if s.resolving[k] {
return "", false
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -178,7 +178,7 @@
"src/main.js": "indexed"
},
"stats": {
"version": "18",
"version": "19",
"files": 4,
"nodes": 7,
"edges": 6,
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -141,7 +141,7 @@
"main.js": "indexed"
},
"stats": {
"version": "18",
"version": "19",
"files": 3,
"nodes": 5,
"edges": 4,
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -141,7 +141,7 @@
"make.js": "indexed"
},
"stats": {
"version": "18",
"version": "19",
"files": 3,
"nodes": 5,
"edges": 4,
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -169,7 +169,7 @@
"src/util.js": "indexed"
},
"stats": {
"version": "18",
"version": "19",
"files": 3,
"nodes": 6,
"edges": 5,
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -281,7 +281,7 @@
"main.js": "indexed"
},
"stats": {
"version": "18",
"version": "19",
"files": 5,
"nodes": 11,
"edges": 10,
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -148,7 +148,7 @@
"lib/src/feature/y.js": "indexed"
},
"stats": {
"version": "18",
"version": "19",
"files": 4,
"nodes": 7,
"edges": 4,
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -125,7 +125,7 @@
"main.js": "indexed"
},
"stats": {
"version": "18",
"version": "19",
"files": 1,
"nodes": 5,
"edges": 3,
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -130,7 +130,7 @@
"packages/core/util.js": "indexed"
},
"stats": {
"version": "18",
"version": "19",
"files": 3,
"nodes": 6,
"edges": 5,
Expand Down
2 changes: 1 addition & 1 deletion projectgraph/testdata/python/all-hides/graph.golden.json
Original file line number Diff line number Diff line change
Expand Up @@ -177,7 +177,7 @@
"pkg/a.py": "indexed"
},
"stats": {
"version": "18",
"version": "19",
"files": 3,
"nodes": 7,
"edges": 6,
Expand Down
2 changes: 1 addition & 1 deletion projectgraph/testdata/python/assignment/graph.golden.json
Original file line number Diff line number Diff line change
Expand Up @@ -71,7 +71,7 @@
"main.py": "indexed"
},
"stats": {
"version": "18",
"version": "19",
"files": 1,
"nodes": 3,
"edges": 2,
Expand Down
Loading
Loading