From bf6ca3883fbee1bdde9973daea4ce6f3392dff9e Mon Sep 17 00:00:00 2001 From: Antoine Toussaint Date: Sat, 8 Aug 2026 16:57:23 -0400 Subject: [PATCH] fix(code): inspect declared Python requirements --- code/code_unit_discovery.go | 2 +- code/code_unit_discovery_test.go | 16 +-- code/python_server.go | 195 +++++++++++++++++++++++++++---- code/python_server_test.go | 78 +++++++++++++ go.mod | 1 + go.sum | 2 + 6 files changed, 264 insertions(+), 30 deletions(-) diff --git a/code/code_unit_discovery.go b/code/code_unit_discovery.go index 42299920..1fe3b637 100644 --- a/code/code_unit_discovery.go +++ b/code/code_unit_discovery.go @@ -46,7 +46,7 @@ type codeUnitAccumulator struct { // precedence for a genuinely co-located polyglot boundary. var codeUnitDeclarations = []codeUnitDeclaration{ {language: "go", runtimeAgent: "go", priority: 10, match: exactManifest("go.mod")}, - {language: "python", runtimeAgent: "python", priority: 20, match: exactManifest("pyproject.toml", "uv.lock", "setup.py", "setup.cfg", "requirements.txt")}, + {language: "python", runtimeAgent: "python", priority: 20, match: exactManifest("pyproject.toml", "uv.lock", "setup.py", "setup.cfg", "requirements.in", "requirements.txt")}, {language: "typescript", runtimeAgent: "nextjs", priority: 30, match: exactManifest("package.json")}, {language: "rust", runtimeAgent: "rust", priority: 40, match: exactManifest("Cargo.toml")}, {language: "swift", runtimeAgent: "swift", priority: 50, match: exactManifest("Package.swift")}, diff --git a/code/code_unit_discovery_test.go b/code/code_unit_discovery_test.go index 137cc618..9f01b5a3 100644 --- a/code/code_unit_discovery_test.go +++ b/code/code_unit_discovery_test.go @@ -15,6 +15,7 @@ func TestDefaultCodeServerDiscoversSupportedAndGenericCodeUnits(t *testing.T) { "src/api/go.mod", "src/worker/pyproject.toml", "src/worker/requirements.txt", + "src/input-only/requirements.in", "src/ads/build.gradle", "src/cart/cart.sln", "src/cart/src/cart.csproj", @@ -41,19 +42,20 @@ func TestDefaultCodeServerDiscoversSupportedAndGenericCodeUnits(t *testing.T) { t.Fatalf("discover code units: %v", err) } units := response.GetDiscoverCodeUnits().GetCodeUnits() - if len(units) != 5 { - t.Fatalf("code units = %d (%+v), want 5 complete boundaries", len(units), units) + if len(units) != 6 { + t.Fatalf("code units = %d (%+v), want 6 complete boundaries", len(units), units) } want := map[string]struct { language, agent string languages []string manifests []string }{ - "src/ads": {language: "jvm", agent: "generic", languages: []string{"jvm"}, manifests: []string{"src/ads/build.gradle"}}, - "src/api": {language: "go", agent: "go", languages: []string{"go"}, manifests: []string{"src/api/go.mod"}}, - "src/cart": {language: "dotnet", agent: "generic", languages: []string{"dotnet"}, manifests: []string{"src/cart/cart.sln", "src/cart/src/cart.csproj", "src/cart/tests/cart.tests.csproj"}}, - "src/mixed": {language: "go", agent: "generic", languages: []string{"go", "typescript"}, manifests: []string{"src/mixed/go.mod", "src/mixed/package.json"}}, - "src/worker": {language: "python", agent: "python", languages: []string{"python"}, manifests: []string{"src/worker/pyproject.toml", "src/worker/requirements.txt"}}, + "src/ads": {language: "jvm", agent: "generic", languages: []string{"jvm"}, manifests: []string{"src/ads/build.gradle"}}, + "src/api": {language: "go", agent: "go", languages: []string{"go"}, manifests: []string{"src/api/go.mod"}}, + "src/cart": {language: "dotnet", agent: "generic", languages: []string{"dotnet"}, manifests: []string{"src/cart/cart.sln", "src/cart/src/cart.csproj", "src/cart/tests/cart.tests.csproj"}}, + "src/mixed": {language: "go", agent: "generic", languages: []string{"go", "typescript"}, manifests: []string{"src/mixed/go.mod", "src/mixed/package.json"}}, + "src/input-only": {language: "python", agent: "python", languages: []string{"python"}, manifests: []string{"src/input-only/requirements.in"}}, + "src/worker": {language: "python", agent: "python", languages: []string{"python"}, manifests: []string{"src/worker/pyproject.toml", "src/worker/requirements.txt"}}, } for _, unit := range units { expected, ok := want[unit.GetPath()] diff --git a/code/python_server.go b/code/python_server.go index 7974b3e4..50fcd577 100644 --- a/code/python_server.go +++ b/code/python_server.go @@ -9,15 +9,17 @@ import ( "os" "os/exec" "path/filepath" + "regexp" "sort" "strings" basev0 "github.com/codefly-dev/core/generated/go/codefly/base/v0" codev0 "github.com/codefly-dev/core/generated/go/codefly/services/code/v0" + piprequirements "github.com/scagogogo/python-requirements-parser/pkg/parser" ) // PythonCodeServer extends DefaultCodeServer with Python-specific operations: -// GetProjectInfo (pyproject.toml + uv) and ListDependencies (uv pip list). +// GetProjectInfo (declared project metadata) and ListDependencies (uv pip list). type PythonCodeServer struct { *DefaultCodeServer } @@ -44,15 +46,16 @@ func (s *PythonCodeServer) handleGetProjectInfo(ctx context.Context, _ *codev0.C resp := &codev0.GetProjectInfoResponse{Language: "python"} var manifestDeps []*codev0.Dependency - // Parse pyproject.toml for module name, Python version, AND - // declared dependencies. Manifest-declared deps land first; - // uv pip list output (if available) merges in next and adds - // transitive deps + resolved versions. + // Parse pyproject.toml for module name, Python version, and declared + // dependencies. Project inspection is intentionally declarative: the + // separate ListDependencies capability owns installed-environment state. data, err := s.FS.ReadFile(filepath.Join(srcDir, "pyproject.toml")) if err == nil { resp.Module, resp.LanguageVersion = parsePyprojectTOML(string(data)) manifestDeps = parsePyprojectDependencies(string(data)) } + requirementDeps, requirementErr := s.declaredRequirementDependencies() + manifestDeps = mergePythonDependencies(manifestDeps, requirementDeps) // Discover packages (directories with __init__.py) resp.Packages = s.discoverPackages(srcDir) @@ -60,31 +63,179 @@ func (s *PythonCodeServer) handleGetProjectInfo(ctx context.Context, _ *codev0.C // File hashes resp.FileHashes = s.computeFileHashes(srcDir) - // Merge: manifest deps (declared, source of truth for direct - // deps) + uv pip list (resolved, includes transitives). Dedup - // by name; manifest wins on conflicts because it carries the - // caller's declared version constraint. - seen := map[string]bool{} - for _, d := range manifestDeps { - if d == nil || d.Name == "" || seen[d.Name] { + resp.Dependencies = manifestDeps + if requirementErr != nil { + return codeFailure(wrapProjectInfoPython(resp), basev0.FailureCode_FAILURE_CODE_VALIDATION_FAILED, "code.get-project-info", requirementErr.Error()), nil + } + resp.SourceFiles, err = inspectSourceImports(ctx, s.FS, srcDir, "python") + if err != nil { + return codeFailure(wrapProjectInfoPython(resp), basev0.FailureCode_FAILURE_CODE_VALIDATION_FAILED, "code.get-project-info", err.Error()), nil + } + + return wrapProjectInfoPython(resp), nil +} + +var pythonDistributionName = regexp.MustCompile(`^[A-Za-z0-9](?:[A-Za-z0-9._-]*[A-Za-z0-9])?$`) + +// declaredRequirementDependencies returns direct dependency evidence from the +// best root requirements declaration. Author-maintained .in input wins over a +// compiled .txt file; referenced requirement files are then followed through +// the Code server VFS so overlays and remote agent filesystems behave exactly +// like local workspaces. +func (s *PythonCodeServer) declaredRequirementDependencies() ([]*codev0.Dependency, error) { + entries, err := s.FS.ReadDir(s.SourceDir) + if err != nil { + return nil, fmt.Errorf("list Python dependency declarations: %w", err) + } + type candidate struct { + name string + rank int + } + var candidates []candidate + for _, entry := range entries { + if entry.IsDir() { continue } - seen[d.Name] = true - resp.Dependencies = append(resp.Dependencies, d) + name := strings.ToLower(entry.Name()) + rank := -1 + switch { + case name == "requirements.in": + rank = 0 + case strings.HasPrefix(name, "requirements") && strings.HasSuffix(name, ".in"): + rank = 1 + case name == "requirements.txt": + rank = 2 + case strings.HasPrefix(name, "requirements") && strings.HasSuffix(name, ".txt"): + rank = 3 + } + if rank >= 0 { + candidates = append(candidates, candidate{name: entry.Name(), rank: rank}) + } } - for _, d := range s.listUVDependencies(srcDir) { - if d == nil || d.Name == "" || seen[d.Name] { - continue + if len(candidates) == 0 { + return nil, nil + } + sort.Slice(candidates, func(i, j int) bool { + if candidates[i].rank != candidates[j].rank { + return candidates[i].rank < candidates[j].rank } - seen[d.Name] = true - resp.Dependencies = append(resp.Dependencies, d) + return candidates[i].name < candidates[j].name + }) + return s.parseRequirementFile(candidates[0].name, make(map[string]bool)) +} + +func (s *PythonCodeServer) parseRequirementFile(relative string, active map[string]bool) ([]*codev0.Dependency, error) { + relative = filepath.Clean(filepath.FromSlash(strings.TrimSpace(relative))) + if relative == "." || filepath.IsAbs(relative) || relative == ".." || strings.HasPrefix(relative, ".."+string(filepath.Separator)) { + return nil, fmt.Errorf("requirements include %q escapes the project root", relative) } - resp.SourceFiles, err = inspectSourceImports(ctx, s.FS, srcDir, "python") + if active[relative] { + return nil, fmt.Errorf("requirements include cycle at %q", filepath.ToSlash(relative)) + } + active[relative] = true + defer delete(active, relative) + + data, err := s.FS.ReadFile(filepath.Join(s.SourceDir, relative)) if err != nil { - return codeFailure(wrapProjectInfoPython(resp), basev0.FailureCode_FAILURE_CODE_VALIDATION_FAILED, "code.get-project-info", err.Error()), nil + return nil, fmt.Errorf("read requirements declaration %q: %w", filepath.ToSlash(relative), err) + } + parsed, err := piprequirements.NewWithOptions(false, false).Parse(bytes.NewReader(data)) + if err != nil { + return nil, fmt.Errorf("parse requirements declaration %q: %w", filepath.ToSlash(relative), err) + } + var dependencies []*codev0.Dependency + for _, requirement := range parsed { + if requirement == nil || requirement.IsEmpty || requirement.IsComment || requirement.IsConstraint || len(requirement.GlobalOptions) > 0 { + continue + } + if requirement.IsFileRef { + referenced := filepath.Join(filepath.Dir(relative), filepath.FromSlash(requirement.FileRef)) + included, includeErr := s.parseRequirementFile(referenced, active) + dependencies = mergePythonDependencies(dependencies, included) + if includeErr != nil { + return dependencies, includeErr + } + continue + } + name := strings.TrimSpace(requirement.Name) + if name == "" { + if directName, directReference, ok := namedPythonDirectReference(requirement.OriginalLine); ok { + if !pythonDistributionName.MatchString(directName) { + return dependencies, fmt.Errorf("requirements declaration %q contains invalid distribution name %q", filepath.ToSlash(relative), directName) + } + dependencies = mergePythonDependencies(dependencies, []*codev0.Dependency{{Name: directName, Version: directReference, Direct: true}}) + continue + } + // Unnamed local paths and URLs are valid pip declarations but do not + // carry a stable distribution identity for dependency evidence. + if requirement.IsLocalPath || requirement.IsURL || requirement.IsVCS || requirement.IsEditable { + continue + } + return dependencies, fmt.Errorf("requirements declaration %q contains an unrecognized entry %q", filepath.ToSlash(relative), requirement.OriginalLine) + } + if !pythonDistributionName.MatchString(name) { + return dependencies, fmt.Errorf("requirements declaration %q contains invalid distribution name %q", filepath.ToSlash(relative), name) + } + version := strings.TrimSpace(requirement.Version) + switch { + case version != "": + case requirement.IsVCS: + version = requirement.VCSType + "+" + requirement.URL + case requirement.IsURL: + version = requirement.URL + default: + if _, reference, ok := strings.Cut(requirement.OriginalLine, " @ "); ok { + version = strings.TrimSpace(strings.SplitN(reference, ";", 2)[0]) + } + } + dependencies = mergePythonDependencies(dependencies, []*codev0.Dependency{{Name: name, Version: version, Direct: true}}) } + return dependencies, nil +} - return wrapProjectInfoPython(resp), nil +func namedPythonDirectReference(line string) (string, string, bool) { + line = strings.TrimSpace(strings.SplitN(line, ";", 2)[0]) + name, reference, ok := strings.Cut(line, " @ ") + name = strings.TrimSpace(name) + reference = strings.TrimSpace(reference) + return name, reference, ok && name != "" && reference != "" +} + +func mergePythonDependencies(groups ...[]*codev0.Dependency) []*codev0.Dependency { + seen := make(map[string]bool) + var merged []*codev0.Dependency + for _, dependencies := range groups { + for _, dependency := range dependencies { + if dependency == nil || strings.TrimSpace(dependency.GetName()) == "" { + continue + } + key := normalizePythonDistributionName(dependency.GetName()) + if seen[key] { + continue + } + seen[key] = true + merged = append(merged, dependency) + } + } + return merged +} + +func normalizePythonDistributionName(name string) string { + name = strings.ToLower(strings.TrimSpace(name)) + var normalized strings.Builder + separator := false + for _, char := range name { + if char == '-' || char == '_' || char == '.' { + separator = true + continue + } + if separator && normalized.Len() > 0 { + normalized.WriteByte('-') + } + separator = false + normalized.WriteRune(char) + } + return normalized.String() } func (s *PythonCodeServer) handleListDependencies(_ context.Context, _ *codev0.CodeRequest) (*codev0.CodeResponse, error) { diff --git a/code/python_server_test.go b/code/python_server_test.go index 7f46dec8..3276285a 100644 --- a/code/python_server_test.go +++ b/code/python_server_test.go @@ -7,6 +7,7 @@ import ( "strings" "testing" + basev0 "github.com/codefly-dev/core/generated/go/codefly/base/v0" codev0 "github.com/codefly-dev/core/generated/go/codefly/services/code/v0" ) @@ -30,6 +31,83 @@ func newPythonProject(t *testing.T, files map[string]string) *PythonCodeServer { return NewPythonCodeServer(dir, nil) } +func TestPythonProjectInfoReadsPreferredInputRequirements(t *testing.T) { + server := newPythonProject(t, map[string]string{ + "requirements.in": `requests[security]>=2.0; python_version >= '3.10' \ + --hash=sha256:abcdef1234567890 +-r requirements/base.txt +-e git+https://example.test/internal.git#egg=internal_lib +remote-wheel @ https://example.test/remote.whl; python_version >= '3.11' +`, + "requirements.txt": "transitive-package==99.0\n", + "requirements/base.txt": "pydantic==2.9.0\nRequests==9.9.9\n", + "app.py": "import requests\n", + }) + response, err := server.Execute(t.Context(), &codev0.CodeRequest{ + Operation: &codev0.CodeRequest_GetProjectInfo{GetProjectInfo: &codev0.GetProjectInfoRequest{}}, + }) + if err != nil { + t.Fatal(err) + } + if response.GetFailure() != nil { + t.Fatalf("project info failure = %+v", response.GetFailure()) + } + var got []string + for _, dependency := range response.GetGetProjectInfo().GetDependencies() { + got = append(got, dependency.GetName()+"@"+dependency.GetVersion()) + if !dependency.GetDirect() { + t.Fatalf("declared dependency is not direct: %+v", dependency) + } + } + want := []string{ + "requests@>=2.0", + "pydantic@==2.9.0", + "internal_lib@git+https://example.test/internal.git", + "remote-wheel@https://example.test/remote.whl", + } + if strings.Join(got, "|") != strings.Join(want, "|") { + t.Fatalf("declared dependencies = %#v, want %#v", got, want) + } +} + +func TestPythonProjectInfoReadsRequirementsTxtOnlyProject(t *testing.T) { + server := newPythonProject(t, map[string]string{ + "requirements.txt": "flask==3.1.2\n", + "app.py": "import flask\n", + }) + response, err := server.Execute(t.Context(), &codev0.CodeRequest{ + Operation: &codev0.CodeRequest_GetProjectInfo{GetProjectInfo: &codev0.GetProjectInfoRequest{}}, + }) + if err != nil { + t.Fatal(err) + } + dependencies := response.GetGetProjectInfo().GetDependencies() + if response.GetFailure() != nil || len(dependencies) != 1 || dependencies[0].GetName() != "flask" || dependencies[0].GetVersion() != "==3.1.2" { + t.Fatalf("requirements.txt project info = %+v failure=%+v", response.GetGetProjectInfo(), response.GetFailure()) + } +} + +func TestPythonProjectInfoReturnsTypedPartialFailureForMissingRequirementInclude(t *testing.T) { + server := newPythonProject(t, map[string]string{ + "requirements.in": "requests==2.32.5\n-r requirements/missing.txt\n", + "app.py": "import requests\n", + }) + response, err := server.Execute(t.Context(), &codev0.CodeRequest{ + Operation: &codev0.CodeRequest_GetProjectInfo{GetProjectInfo: &codev0.GetProjectInfoRequest{}}, + }) + if err != nil { + t.Fatal(err) + } + info := response.GetGetProjectInfo() + if info == nil || len(info.GetDependencies()) != 1 || info.GetDependencies()[0].GetName() != "requests" { + t.Fatalf("partial project evidence = %+v", info) + } + if response.GetFailure().GetCode() != basev0.FailureCode_FAILURE_CODE_VALIDATION_FAILED || + !strings.Contains(response.GetFailure().GetMessage(), "requirements/missing.txt") { + t.Fatalf("typed requirement failure = %+v", response.GetFailure()) + } +} + // packagesByPath indexes a GetProjectInfo package list by RelativePath. func packagesByPath(t *testing.T, s *PythonCodeServer) map[string]pkgView { t.Helper() diff --git a/go.mod b/go.mod index 68fff462..64d2cdae 100644 --- a/go.mod +++ b/go.mod @@ -38,6 +38,7 @@ require ( github.com/pelletier/go-toml/v2 v2.4.3 github.com/pkg/errors v0.9.1 github.com/santhosh-tekuri/jsonschema/v6 v6.0.2 + github.com/scagogogo/python-requirements-parser v0.0.0-20250717025652-6ca77234c827 github.com/shirou/gopsutil/v3 v3.24.5 github.com/smacker/go-tree-sitter v0.0.0-20240827094217-dd81d9e9be82 github.com/stretchr/testify v1.11.1 diff --git a/go.sum b/go.sum index 30f8c7c1..74548844 100644 --- a/go.sum +++ b/go.sum @@ -313,6 +313,8 @@ github.com/rogpeppe/go-internal v1.14.1 h1:UQB4HGPB6osV0SQTLymcB4TgvyWu6ZyliaW0t github.com/rogpeppe/go-internal v1.14.1/go.mod h1:MaRKkUm5W0goXpeCfT7UZI6fk/L7L7so1lCWt35ZSgc= github.com/santhosh-tekuri/jsonschema/v6 v6.0.2 h1:KRzFb2m7YtdldCEkzs6KqmJw4nqEVZGK7IN2kJkjTuQ= github.com/santhosh-tekuri/jsonschema/v6 v6.0.2/go.mod h1:JXeL+ps8p7/KNMjDQk3TCwPpBy0wYklyWTfbkIzdIFU= +github.com/scagogogo/python-requirements-parser v0.0.0-20250717025652-6ca77234c827 h1:+Fu0RVDu7WxwtkELZ9V0PjG+HdxL+ANkGKa6gp45J/g= +github.com/scagogogo/python-requirements-parser v0.0.0-20250717025652-6ca77234c827/go.mod h1:3rCK5dhjXuiTnMXVkO2n9eWZ21qWreUtptCmCT9+rNg= github.com/sergi/go-diff v1.4.0 h1:n/SP9D5ad1fORl+llWyN+D6qoUETXNZARKjyY2/KVCw= github.com/sergi/go-diff v1.4.0/go.mod h1:A0bzQcvG0E7Rwjx0REVgAGH58e96+X0MeOfepqsbeW4= github.com/shirou/gopsutil/v3 v3.24.5 h1:i0t8kL+kQTvpAYToeuiVk3TgDeKOFioZO3Ztz/iZ9pI=