feat(gateway): expose agent semantic projections

go.mod

@@ -11,7 +11,7 @@ require (
 	github.com/asottile/dockerfile v3.1.0+incompatible
 	github.com/blang/semver v3.5.1+incompatible
 	github.com/briandowns/spinner v1.23.2
-	github.com/codefly-dev/core v0.2.75
+	github.com/codefly-dev/core v0.2.80
 	github.com/codefly-dev/golor v0.1.3
 	github.com/codefly-dev/llm v0.1.0
 	github.com/codefly-dev/sdk-go v0.1.58
@@ -166,6 +166,7 @@ require (
 	github.com/power-devops/perfstat v0.0.0-20240221224432-82ca36839d55 // indirect
 	github.com/rivo/uniseg v0.4.7 // indirect
 	github.com/santhosh-tekuri/jsonschema/v6 v6.0.2 // indirect
+	github.com/scagogogo/python-requirements-parser v0.0.0-20250717025652-6ca77234c827 // indirect
 	github.com/sergi/go-diff v1.4.0 // indirect
 	github.com/shirou/gopsutil/v3 v3.24.5 // indirect
 	github.com/shoenig/go-m1cpu v0.2.1 // indirect

go.sum

@@ -95,8 +95,8 @@ github.com/clipperhouse/uax29/v2 v2.7.0 h1:+gs4oBZ2gPfVrKPthwbMzWZDaAFPGYK72F0NJ
 github.com/clipperhouse/uax29/v2 v2.7.0/go.mod h1:EFJ2TJMRUaplDxHKj1qAEhCtQPW2tJSwu5BF98AuoVM=
 github.com/cloudflare/circl v1.6.3 h1:9GPOhQGF9MCYUeXyMYlqTR6a5gTrgR/fBLXvUgtVcg8=
 github.com/cloudflare/circl v1.6.3/go.mod h1:2eXP6Qfat4O/Yhh8BznvKnJ+uzEoTQ6jVKJRn81BiS4=
-github.com/codefly-dev/core v0.2.75 h1:57SDudHu08yKUW+hZusNLVLiT2aXIi6BIHsu7Zptyvs=
-github.com/codefly-dev/core v0.2.75/go.mod h1:VXuESK4SxVie0+ffHfOhw+zg3AS3X/tEVkMEcQBXQxY=
+github.com/codefly-dev/core v0.2.80 h1:RwBAWd+7J2ZHHY5RSllfPNlr1wP9CNTmVFQz9UM3rog=
+github.com/codefly-dev/core v0.2.80/go.mod h1:hNxTk7ZnR5AU8imfvNJcGpoULQ1uzML5M3E4/RffWU4=
 github.com/codefly-dev/golor v0.1.3 h1:xmo+ceyJFRYZdvpWE2fNd0jeaadp/Ibm1BnganiGKOc=
 github.com/codefly-dev/golor v0.1.3/go.mod h1:sl/u/K1l7J0Pr3xyVZp8fOJYQItKKst1No9JqgzLLoY=
 github.com/codefly-dev/gortk v0.2.0 h1:7bOlS5valYz2zil+fZctQNcPCYBcPj86abcw9N8h1hQ=
@@ -365,6 +365,8 @@ github.com/rs/cors v1.11.1/go.mod h1:XyqrcTp5zjWr1wsJ8PIRZssZ8b/WMcMf71DJnit4EMU
 github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
 github.com/santhosh-tekuri/jsonschema/v6 v6.0.2 h1:KRzFb2m7YtdldCEkzs6KqmJw4nqEVZGK7IN2kJkjTuQ=
 github.com/santhosh-tekuri/jsonschema/v6 v6.0.2/go.mod h1:JXeL+ps8p7/KNMjDQk3TCwPpBy0wYklyWTfbkIzdIFU=
+github.com/scagogogo/python-requirements-parser v0.0.0-20250717025652-6ca77234c827 h1:+Fu0RVDu7WxwtkELZ9V0PjG+HdxL+ANkGKa6gp45J/g=
+github.com/scagogogo/python-requirements-parser v0.0.0-20250717025652-6ca77234c827/go.mod h1:3rCK5dhjXuiTnMXVkO2n9eWZ21qWreUtptCmCT9+rNg=
 github.com/sergi/go-diff v1.4.0 h1:n/SP9D5ad1fORl+llWyN+D6qoUETXNZARKjyY2/KVCw=
 github.com/sergi/go-diff v1.4.0/go.mod h1:A0bzQcvG0E7Rwjx0REVgAGH58e96+X0MeOfepqsbeW4=
 github.com/shirou/gopsutil/v3 v3.24.5 h1:i0t8kL+kQTvpAYToeuiVk3TgDeKOFioZO3Ztz/iZ9pI=

pkg/engine/service.go

@@ -9,6 +9,7 @@ import (
 	builderv0 "github.com/codefly-dev/core/generated/go/codefly/services/builder/v0"
 	codev0 "github.com/codefly-dev/core/generated/go/codefly/services/code/v0"
 	runtimev0 "github.com/codefly-dev/core/generated/go/codefly/services/runtime/v0"
+	toolingv0 "github.com/codefly-dev/core/generated/go/codefly/services/tooling/v0"
 	"google.golang.org/grpc/codes"
 	"google.golang.org/grpc/status"
 	"google.golang.org/protobuf/proto"
@@ -59,6 +60,19 @@ func (s *Service) ExecuteCode(ctx context.Context, request *codev0.CodeRequest)
 	return response, err
 }
 
+// GetSemanticIndex invokes the production agent's Tooling boundary. Project
+// bytes and parser selection remain inside the agent; callers receive only the
+// shared body-free semantic contract.
+func (s *Service) GetSemanticIndex(ctx context.Context, request *toolingv0.GetSemanticIndexRequest) (*toolingv0.GetSemanticIndexResponse, error) {
+	var response *toolingv0.GetSemanticIndexResponse
+	err := s.withSession(ctx, "tooling.GetSemanticIndex", true, func(session *AgentSession) error {
+		var err error
+		response, err = session.tooling.GetSemanticIndex(ctx, request)
+		return err
+	})
+	return response, err
+}
+
 // Build invokes the service agent's runtime build behavior.
 func (s *Service) Build(ctx context.Context, request *runtimev0.BuildRequest) (*runtimev0.BuildResponse, error) {
 	var response *runtimev0.BuildResponse

pkg/engine/supervisor.go

@@ -14,6 +14,7 @@ import (
 	builderv0 "github.com/codefly-dev/core/generated/go/codefly/services/builder/v0"
 	codev0 "github.com/codefly-dev/core/generated/go/codefly/services/code/v0"
 	runtimev0 "github.com/codefly-dev/core/generated/go/codefly/services/runtime/v0"
+	toolingv0 "github.com/codefly-dev/core/generated/go/codefly/services/tooling/v0"
 	"github.com/codefly-dev/core/network"
 	"github.com/codefly-dev/core/resources"
 	"google.golang.org/grpc/connectivity"
@@ -49,6 +50,7 @@ type AgentSession struct {
 	agentAPI   agentv0.AgentClient
 	builder    builderv0.BuilderClient
 	runtime    runtimev0.RuntimeClient
+	tooling    toolingv0.ToolingClient
 	builderMu  sync.Mutex
 	builderOK  bool
 	closeOnce  sync.Once
@@ -162,6 +164,7 @@ func (s *AgentSupervisor) acquire(ctx context.Context, target ServiceTarget) (*A
 		agentAPI:   agentv0.NewAgentClient(connection.GRPCConn()),
 		builder:    builderv0.NewBuilderClient(connection.GRPCConn()),
 		runtime:    runtimev0.NewRuntimeClient(connection.GRPCConn()),
+		tooling:    toolingv0.NewToolingClient(connection.GRPCConn()),
 	}
 	if err := initializeRuntime(loadCtx, session); err != nil {
 		connection.Close()

pkg/gateway/code_unit_project_info_integration_test.go

@@ -4,6 +4,7 @@ import (
 	"reflect"
 	"testing"
 
+	basev0 "github.com/codefly-dev/core/generated/go/codefly/base/v0"
 	gatewayv1 "github.com/codefly-dev/core/generated/go/mind/gateway/v1"
 )
 
@@ -40,26 +41,30 @@ class Ads {}
 	t.Cleanup(func() { _ = server.Close() })
 
 	tests := []struct {
-		name         string
-		target       *gatewayv1.CodeUnitTarget
-		language     string
-		module       string
-		version      string
-		dependencies []string
-		sourcePath   string
-		imports      []string
+		name             string
+		target           *gatewayv1.CodeUnitTarget
+		language         string
+		module           string
+		version          string
+		dependencies     []string
+		sourcePath       string
+		imports          []string
+		semanticLanguage string
+		semanticSymbol   string
 	}{
 		{
 			name: "jvm", target: &gatewayv1.CodeUnitTarget{Id: "shop/ads", Path: "src/ads"},
 			language: "jvm", module: "example.ads", version: "21",
 			dependencies: []string{"io.grpc:grpc-protobuf@1.82.1"},
 			sourcePath:   "src/ads/src/main/java/example/Ads.java", imports: []string{"io.grpc.Server"},
+			semanticLanguage: "java", semanticSymbol: "Ads",
 		},
 		{
 			name: "dotnet", target: &gatewayv1.CodeUnitTarget{Id: "shop/cart", Path: "src/cart"},
 			language: "dotnet", module: "cartservice", version: "net10.0",
 			dependencies: []string{"Grpc.AspNetCore@2.80.0"},
 			sourcePath:   "src/cart/src/Program.cs", imports: []string{"Grpc.Core"},
+			semanticLanguage: "csharp", semanticSymbol: "Program",
 		},
 	}
 	for _, test := range tests {
@@ -90,6 +95,37 @@ class Ads {}
 			if _, ok := response.GetFileHashes()[test.sourcePath]; !ok {
 				t.Fatalf("repository-relative source hash %q missing from %+v", test.sourcePath, response.GetFileHashes())
 			}
+
+			semantic, err := server.GetSemanticIndex(t.Context(), &gatewayv1.GetSemanticIndexRequest{CodeUnit: test.target})
+			if err != nil {
+				t.Fatal(err)
+			}
+			if semantic.GetFailure() != nil {
+				t.Fatalf("semantic-index failure = %+v", semantic.GetFailure())
+			}
+			if !reflect.DeepEqual(semantic.GetCodeUnit(), test.target) {
+				t.Fatalf("semantic code unit = %+v, want %+v", semantic.GetCodeUnit(), test.target)
+			}
+			index := semantic.GetIndex()
+			if index.GetState() != basev0.SemanticIndexState_SEMANTIC_INDEX_STATE_COMPLETE ||
+				!reflect.DeepEqual(index.GetLanguages(), []string{test.semanticLanguage}) {
+				t.Fatalf("semantic coverage = state %s languages %v issues %+v", index.GetState(), index.GetLanguages(), index.GetIssues())
+			}
+			if index.GetAnalyzer() == "" || index.GetAnalyzerVersion() == "" {
+				t.Fatalf("semantic analyzer provenance is incomplete: %+v", index)
+			}
+			if len(index.GetFiles()) != 1 || index.GetFiles()[0].GetPath() != test.sourcePath {
+				t.Fatalf("semantic files = %+v, want repository-relative %q", index.GetFiles(), test.sourcePath)
+			}
+			foundSymbol := false
+			for _, symbol := range index.GetSymbols() {
+				if symbol.GetName() == test.semanticSymbol && symbol.GetLocation().GetPath() == test.sourcePath {
+					foundSymbol = true
+				}
+			}
+			if !foundSymbol {
+				t.Fatalf("semantic symbols = %+v, want %q in %q", index.GetSymbols(), test.semanticSymbol, test.sourcePath)
+			}
 		})
 	}
 }

pkg/gateway/code_unit_runtime.go

@@ -11,6 +11,7 @@ import (
 	"unicode/utf8"
 
 	"github.com/codefly-dev/cli/pkg/engine"
+	basev0 "github.com/codefly-dev/core/generated/go/codefly/base/v0"
 	runtimev0 "github.com/codefly-dev/core/generated/go/codefly/services/runtime/v0"
 	gatewayv1 "github.com/codefly-dev/core/generated/go/mind/gateway/v1"
 	selectioncontract "github.com/codefly-dev/core/runners/testselection"
@@ -420,6 +421,44 @@ func rebaseCodeUnitFile(unitPath, file string) string {
 	return path.Join(unitPath, clean)
 }
 
+// rebaseCodeUnitSemanticIndex converts the agent-root-relative path contract
+// into the repository-relative Gateway contract without changing analyzer
+// facts or semantic identities.
+func rebaseCodeUnitSemanticIndex(unitPath string, index *basev0.SemanticIndex) *basev0.SemanticIndex {
+	if index == nil {
+		return nil
+	}
+	result, ok := proto.Clone(index).(*basev0.SemanticIndex)
+	if !ok {
+		return nil
+	}
+	for _, file := range result.GetFiles() {
+		file.Path = rebaseCodeUnitFile(unitPath, file.GetPath())
+	}
+	for _, symbol := range result.GetSymbols() {
+		rebaseSemanticLocation(unitPath, symbol.GetLocation())
+		for _, use := range symbol.GetCalls() {
+			rebaseSemanticLocation(unitPath, use.GetLocation())
+		}
+		for _, use := range symbol.GetReferences() {
+			rebaseSemanticLocation(unitPath, use.GetLocation())
+		}
+	}
+	for _, issue := range result.GetIssues() {
+		if strings.TrimSpace(issue.GetPath()) != "" {
+			issue.Path = rebaseCodeUnitFile(unitPath, issue.GetPath())
+		}
+	}
+	return result
+}
+
+func rebaseSemanticLocation(unitPath string, location *basev0.SemanticLocation) {
+	if location == nil || strings.TrimSpace(location.GetPath()) == "" {
+		return
+	}
+	location.Path = rebaseCodeUnitFile(unitPath, location.GetPath())
+}
+
 func aggregateCodeUnitTestResponses(request *runtimev0.TestRequest, results []codeUnitTestResult) *runtimev0.TestResponse {
 	counts := &runtimev0.TestCounts{}
 	truncation := &runtimev0.TestTruncation{}

pkg/gateway/code_unit_runtime_test.go

@@ -5,10 +5,44 @@ import (
 	"testing"
 	"unicode/utf8"
 
+	basev0 "github.com/codefly-dev/core/generated/go/codefly/base/v0"
 	runtimev0 "github.com/codefly-dev/core/generated/go/codefly/services/runtime/v0"
 	"google.golang.org/protobuf/proto"
 )
 
+func TestRebaseCodeUnitSemanticIndexPreservesFactsAndRebasesEveryPath(t *testing.T) {
+	original := &basev0.SemanticIndex{
+		State: basev0.SemanticIndexState_SEMANTIC_INDEX_STATE_DEGRADED,
+		Files: []*basev0.SemanticFile{{Path: "src/main.go", ContentSha256: "abc"}},
+		Symbols: []*basev0.SemanticSymbol{{
+			QualifiedName: "demo.Main", Location: &basev0.SemanticLocation{Path: "src/main.go", StartLine: 3},
+			Calls:      []*basev0.SemanticUse{{Name: "Run", Location: &basev0.SemanticLocation{Path: "src/main.go", StartLine: 4}}},
+			References: []*basev0.SemanticUse{{Name: "Config", Location: &basev0.SemanticLocation{Path: "src/config.go", StartLine: 5}}},
+		}},
+		Issues: []*basev0.SemanticIssue{{Code: "parse_failed", Path: "src/broken.go"}, {Code: "unsupported_source"}},
+	}
+
+	rebased := rebaseCodeUnitSemanticIndex("services/api", original)
+	if got := rebased.GetFiles()[0].GetPath(); got != "services/api/src/main.go" {
+		t.Fatalf("file path = %q", got)
+	}
+	symbol := rebased.GetSymbols()[0]
+	if symbol.GetLocation().GetPath() != "services/api/src/main.go" ||
+		symbol.GetCalls()[0].GetLocation().GetPath() != "services/api/src/main.go" ||
+		symbol.GetReferences()[0].GetLocation().GetPath() != "services/api/src/config.go" {
+		t.Fatalf("semantic locations were not rebased: %#v", symbol)
+	}
+	if rebased.GetIssues()[0].GetPath() != "services/api/src/broken.go" || rebased.GetIssues()[1].GetPath() != "" {
+		t.Fatalf("issue paths = %#v", rebased.GetIssues())
+	}
+	if original.GetFiles()[0].GetPath() != "src/main.go" || original.GetSymbols()[0].GetLocation().GetPath() != "src/main.go" {
+		t.Fatal("rebasing mutated the agent response")
+	}
+	if rebased.GetState() != original.GetState() || rebased.GetFiles()[0].GetContentSha256() != "abc" {
+		t.Fatal("rebasing changed semantic facts")
+	}
+}
+
 // TestPackageSelectionForwardsPluginPackageIdentityUnchanged pins the contract
 // that a package selection routed to a unit is forwarded verbatim.
 // TestPackageSelection.package is the plugin's project-level package identity

pkg/gateway/server.go

@@ -45,6 +45,7 @@ import (
 	builderv0 "github.com/codefly-dev/core/generated/go/codefly/services/builder/v0"
 	codev0 "github.com/codefly-dev/core/generated/go/codefly/services/code/v0"
 	runtimev0 "github.com/codefly-dev/core/generated/go/codefly/services/runtime/v0"
+	toolingv0 "github.com/codefly-dev/core/generated/go/codefly/services/tooling/v0"
 	gatewayv1 "github.com/codefly-dev/core/generated/go/mind/gateway/v1"
 	githubtoolbox "github.com/codefly-dev/core/toolbox/github"
 	"github.com/codefly-dev/core/wool"
@@ -173,6 +174,7 @@ type Server struct {
 // Gateway adapter. engine.Service is the production implementation.
 type serviceExecution interface {
 	ExecuteCode(context.Context, *codev0.CodeRequest) (*codev0.CodeResponse, error)
+	GetSemanticIndex(context.Context, *toolingv0.GetSemanticIndexRequest) (*toolingv0.GetSemanticIndexResponse, error)
 	Build(context.Context, *runtimev0.BuildRequest) (*runtimev0.BuildResponse, error)
 	Test(context.Context, *runtimev0.TestRequest) (*runtimev0.TestResponse, error)
 	Lint(context.Context, *runtimev0.LintRequest) (*runtimev0.LintResponse, error)
@@ -1343,6 +1345,46 @@ func (s *Server) GetProjectInfo(ctx context.Context, req *gatewayv1.GetProjectIn
 	return response, nil
 }
 
+// GetSemanticIndex binds one exact code-unit root and invokes the selected
+// production agent's Tooling service. The Gateway only rebases locations; it
+// never chooses a parser or interprets project syntax.
+func (s *Server) GetSemanticIndex(ctx context.Context, req *gatewayv1.GetSemanticIndexRequest) (*gatewayv1.GetSemanticIndexResponse, error) {
+	requestedService := ""
+	if req != nil {
+		requestedService = req.GetService()
+	}
+	if err := s.validateService(requestedService); err != nil {
+		return nil, err
+	}
+	if req == nil || req.GetCodeUnit() == nil {
+		return &gatewayv1.GetSemanticIndexResponse{Failure: failures.New(basev0.FailureCode_FAILURE_CODE_INVALID_ARGUMENT, "gateway.get-semantic-index", "code_unit is required")}, nil
+	}
+	targets, err := s.normalizeCodeUnitTargets([]*gatewayv1.CodeUnitTarget{req.GetCodeUnit()})
+	if err != nil {
+		return &gatewayv1.GetSemanticIndexResponse{CodeUnit: cloneCodeUnitTarget(req.GetCodeUnit()), Failure: failures.New(basev0.FailureCode_FAILURE_CODE_INVALID_ARGUMENT, "gateway.get-semantic-index", err.Error())}, nil
+	}
+	target := targets[0]
+	inspected := &gatewayv1.CodeUnitTarget{Id: target.id, Path: target.path}
+	service, err := s.serviceBehaviorForCodeUnit(target, "")
+	if err != nil {
+		return &gatewayv1.GetSemanticIndexResponse{CodeUnit: inspected, Failure: gatewaySemanticIndexFailure(err)}, nil
+	}
+	response, err := service.GetSemanticIndex(ctx, &toolingv0.GetSemanticIndexRequest{})
+	if err != nil {
+		return &gatewayv1.GetSemanticIndexResponse{CodeUnit: inspected, Failure: gatewaySemanticIndexFailure(err)}, nil
+	}
+	if response.GetIndex() == nil {
+		return &gatewayv1.GetSemanticIndexResponse{
+			CodeUnit: inspected,
+			Failure:  failures.Ensure(response.GetFailure(), basev0.FailureCode_FAILURE_CODE_INTERNAL, "gateway.get-semantic-index", "agent returned no semantic index"),
+		}, nil
+	}
+	return &gatewayv1.GetSemanticIndexResponse{
+		Index:   rebaseCodeUnitSemanticIndex(target.path, response.GetIndex()),
+		Failure: failures.Clone(response.GetFailure()), CodeUnit: inspected,
+	}, nil
+}
+
 func cloneCodeUnitTarget(target *gatewayv1.CodeUnitTarget) *gatewayv1.CodeUnitTarget {
 	if target == nil {
 		return nil
@@ -1357,6 +1399,13 @@ func gatewayProjectInfoFailure(err error) *basev0.Failure {
 	return failures.FromError("gateway.get-project-info", err)
 }
 
+func gatewaySemanticIndexFailure(err error) *basev0.Failure {
+	if failure, ok := failures.Extract(err); ok {
+		return failures.Clone(failure)
+	}
+	return failures.FromError("gateway.get-semantic-index", err)
+}
+
 // DiscoverCodeUnits serves Codefly's language-neutral structural inventory
 // directly from the rooted source behavior. Discovery must not first select or
 // launch a language plugin: the inventory is what downstream consumers use to

pkg/gateway/server_test.go

@@ -32,6 +32,7 @@ import (
 	agentv0 "github.com/codefly-dev/core/generated/go/codefly/services/agent/v0"
 	codev0 "github.com/codefly-dev/core/generated/go/codefly/services/code/v0"
 	runtimev0 "github.com/codefly-dev/core/generated/go/codefly/services/runtime/v0"
+	toolingv0 "github.com/codefly-dev/core/generated/go/codefly/services/tooling/v0"
 	gatewayv1 "github.com/codefly-dev/core/generated/go/mind/gateway/v1"
 	codefly "github.com/codefly-dev/sdk-go"
 	"google.golang.org/grpc"
@@ -62,12 +63,20 @@ type mockServiceExecution struct {
 	code    codev0.CodeClient
 	runtime runtimev0.RuntimeClient
 	agent   agentv0.AgentClient
+	tooling toolingv0.ToolingClient
 }
 
 func (m *mockServiceExecution) ExecuteCode(ctx context.Context, request *codev0.CodeRequest) (*codev0.CodeResponse, error) {
 	return m.code.Execute(ctx, request)
 }
 
+func (m *mockServiceExecution) GetSemanticIndex(ctx context.Context, request *toolingv0.GetSemanticIndexRequest) (*toolingv0.GetSemanticIndexResponse, error) {
+	if m.tooling == nil {
+		return nil, fmt.Errorf("GetSemanticIndex not configured")
+	}
+	return m.tooling.GetSemanticIndex(ctx, request)
+}
+
 func (m *mockServiceExecution) Build(ctx context.Context, request *runtimev0.BuildRequest) (*runtimev0.BuildResponse, error) {
 	return m.runtime.Build(ctx, request)
 }

pkg/sourceworkspace/source.go

@@ -16,15 +16,15 @@ import (
 )
 
 const (
-	GenericGoPluginVersion     = "0.0.24"
-	GenericPythonPluginVersion = "0.0.28"
-	GenericPluginVersion       = "0.0.18"
+	GenericGoPluginVersion     = "0.0.25"
+	GenericPythonPluginVersion = "0.0.29"
+	GenericPluginVersion       = "0.0.19"
 	// NodePluginVersion is published under the historical nextjs agent name,
 	// but owns generic Node.js/TypeScript validation as well as Next.js-specific
 	// lifecycle behavior selected from the package manifest.
-	NodePluginVersion  = "0.0.131"
-	RustPluginVersion  = "0.0.18"
-	SwiftPluginVersion = "0.0.13"
+	NodePluginVersion  = "0.0.132"
+	RustPluginVersion  = "0.0.22"
+	SwiftPluginVersion = "0.0.14"
 	pythonSetupMarker  = "setup.py"
 )