Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 5 additions & 1 deletion distgraph/graph.go
Original file line number Diff line number Diff line change
Expand Up @@ -317,7 +317,11 @@ func newBuilder(ctx context.Context, in Input) (*builder, error) {
found, problems := modules.PythonDistributions(sp)
b.problems = append(b.problems, problems...)
for _, d := range found {
b.addDistribution(sp, d)
// The code of an editable install is the project, which the
// walk of the project reads.
if !d.Editable {
b.addDistribution(sp, d)
Comment on lines +322 to +323

Copy link
Copy Markdown

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

P2 Badge Retain editable dependencies outside the scanned project

When the environment contains an editable dependency installed from another directory (for example, a sibling library), Editable does not mean its source is inside ProjectFS. Dropping that distribution also removes its import names from byTop, while the project walk never reads its code. If the project imports that library inside a try guarded by ImportError, resolve now ignores the import entirely instead of reaching the library's open node, so the library's transitive dependencies can incorrectly appear unreachable. Only omit an editable distribution after establishing that the project walk covers its source; otherwise retain it as an open node.

Useful? React with 👍 / 👎.

}
}
}
b.stats.Distributions = len(b.order) - 1
Expand Down
11 changes: 11 additions & 0 deletions distgraph/graph_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -292,6 +292,17 @@ func TestBuildSkipsTheEnvironmentInsideTheProject(t *testing.T) {
assert.Equal(t, 2, g.Stats.Files)
}

func TestBuildLeavesOutAnEditableInstall(t *testing.T) {
project := fstest.MapFS{"app/__init__.py": py("requests"), "main.py": py("app")}
sp := sitePackages(nil, distInfo{name: "requests", version: "2.31.0", files: []string{"requests/__init__.py"}}, distInfo{name: "app", version: "0.1.0", files: []string{"__editable__.app-0.1.0.pth"}})
sp["app-0.1.0.dist-info/direct_url.json"] = text(`{"url": "file:///src/app", "dir_info": {"editable": true}}`)
g := build(t, Input{ProjectFS: project, SitePackages: []fs.FS{sp}})
_, ok := g.Node("app")
assert.False(t, ok, "the editable install is the project")
assert.Equal(t, []string{"project -> requests"}, edgeList(g))
assert.Equal(t, 1, g.Stats.Distributions)
}

func TestBuildRejectsABadInput(t *testing.T) {
_, err := Build(context.Background(), Input{})
assert.EqualError(t, err, "distgraph: no project")
Expand Down
3 changes: 2 additions & 1 deletion docs/distgraph.md
Original file line number Diff line number Diff line change
Expand Up @@ -63,7 +63,8 @@ A distribution is open when:
of `METADATA`, the import names of `top_level.txt` or `RECORD`, the
extension modules of `RECORD` and the `Requires-Dist` lines with no
`extra` marker. A `.dist-info` that it cannot read is a problem of the
graph, not a node.
graph, not a node. An editable install, whose `direct_url.json` says
so, is the project itself, so it is no node either.
2. It reads each Python file of the project, its test files too, except
under a hidden directory, a `__pycache__`, a `node_modules`, a
`site-packages` or a virtual environment, which holds a `pyvenv.cfg`.
Expand Down
31 changes: 30 additions & 1 deletion projectgraph/modules/pydist.go
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,9 @@ import (
"bufio"
"bytes"
"encoding/csv"
"encoding/json"
"errors"
"fmt"
"io"
"io/fs"
"path"
Expand All @@ -29,6 +31,10 @@ type Distribution struct {
// Requires holds the names of the distributions that the Requires-Dist
// lines of METADATA name with no extra marker, in order, each once.
Requires []string
// Editable marks an editable install, whose direct_url.json says so.
// Its code is the project that pip or uv installed from, not a copy
// under the root.
Editable bool
}

// Unit returns the package unit of the distribution, as pypi:PyYAML@6.0.1.
Expand Down Expand Up @@ -215,7 +221,30 @@ func readDistribution(root fs.FS, dir string) (Distribution, error) {
if len(d.TopLevel) == 0 {
d.TopLevel = topLevelOf(d.Files, d.Compiled)
}
return d, nil
d.Editable, err = editableInstall(root, dir)
return d, err
}

// editableInstall reads the direct_url.json of a .dist-info, which PEP 610
// writes for an install from a directory or a URL, and reports an editable
// install. A .dist-info with no such file is a regular install.
func editableInstall(root fs.FS, dir string) (bool, error) {
data, err := fs.ReadFile(root, path.Join(dir, "direct_url.json"))
if errors.Is(err, fs.ErrNotExist) {
return false, nil
}
if err != nil {
return false, err
}
var u struct {
DirInfo struct {
Editable bool `json:"editable"`
} `json:"dir_info"`
}
if err := json.Unmarshal(data, &u); err != nil {
return false, fmt.Errorf("direct_url.json: %w", err)
}
return u.DirInfo.Editable, nil
}

// readRecord returns the files of a RECORD under the root, and the module
Expand Down
15 changes: 12 additions & 3 deletions projectgraph/modules/pydist_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,11 @@ func TestPythonDistributions(t *testing.T) {
"six-1.16.0.dist-info/RECORD": {Data: []byte("six.py,sha256=abc,34549\nsix-1.16.0.dist-info/RECORD,,\n")},
"six-1.16.0.dist-info/top_level.txt": {Data: []byte("six\n")},
"google_auth-2.0.0.dist-info/METADATA": {Data: []byte("Name: google-auth\nVersion: 2.0.0\n")},
"app-0.1.0.dist-info/METADATA": {Data: []byte("Name: app\nVersion: 0.1.0\n")},
"app-0.1.0.dist-info/direct_url.json": {Data: []byte(`{"url": "file:///src/app", "dir_info": {"editable": true}}`)},
"app-0.1.0.dist-info/RECORD": {Data: []byte("__editable__.app-0.1.0.pth,,\n")},
"lib-0.2.0.dist-info/METADATA": {Data: []byte("Name: lib\nVersion: 0.2.0\n")},
"lib-0.2.0.dist-info/direct_url.json": {Data: []byte(`{"url": "https://example.com/lib.git", "vcs_info": {"vcs": "git"}}`)},
"google_auth-2.0.0.dist-info/RECORD": {Data: []byte("google/auth/__init__.py,,\ngoogle/auth/crypt.py,,\ngoogle/oauth2/__init__.py,,\n")},
"nometa.dist-info/RECORD": {Data: []byte("x.py,,\n")},
"broken.dist-info/METADATA": {Data: []byte("Version: 1\n")},
Expand All @@ -31,7 +36,7 @@ func TestPythonDistributions(t *testing.T) {
dists, problems := PythonDistributions(root)
require.Len(t, problems, 1, "a .dist-info with no METADATA is no distribution, and one with no Name is a problem")
assert.Equal(t, "broken.dist-info", problems[0].Path)
require.Len(t, dists, 3)
require.Len(t, dists, 5)

yaml := dists[0]
assert.Equal(t, "PyYAML", yaml.Name)
Expand All @@ -43,11 +48,15 @@ func TestPythonDistributions(t *testing.T) {
assert.Equal(t, []string{"_yaml", "yaml"}, yaml.Roots())
assert.Equal(t, []string{"six", "typing_extensions", "google-auth"}, yaml.Requires, "a requirement with an extra marker is no requirement of a plain install")

google := dists[1]
assert.True(t, dists[1].Editable, "app is an editable install")
assert.False(t, dists[3].Editable, "an install from a git URL is no editable install")
assert.False(t, yaml.Editable)

google := dists[2]
assert.Equal(t, []string{"google"}, google.TopLevel)
assert.Equal(t, []string{"google/auth", "google/oauth2"}, google.Roots(), "a namespace package gives the packages under it")

six := dists[2]
six := dists[4]
assert.Equal(t, []string{"six"}, six.TopLevel)
assert.Equal(t, []string{"six.py"}, six.Roots())

Expand Down
Loading