From 8579e2113f6a07c967e351bbf4505d4e70d7b839 Mon Sep 17 00:00:00 2001 From: Sarah Asad Date: Wed, 10 Jun 2026 16:36:31 -0700 Subject: [PATCH 01/13] remove system-requirements-lock.txt --- amber/system-requirements-lock.txt | 102 ----------------------------- 1 file changed, 102 deletions(-) delete mode 100644 amber/system-requirements-lock.txt diff --git a/amber/system-requirements-lock.txt b/amber/system-requirements-lock.txt deleted file mode 100644 index 29f67d38222..00000000000 --- a/amber/system-requirements-lock.txt +++ /dev/null @@ -1,102 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -# This file is manually generated to track system packages used in PVEs. -# NOTE: This file must be updated whenever requirements.txt or -# operator-requirements.txt changes. - -aiohappyeyeballs==2.6.1 -aiohttp==3.13.5 -aioitertools==0.13.0 -aiobotocore==2.25.1 -aiosignal==1.4.0 -annotated-types==0.7.0 -appdirs==1.4.4 -asn1crypto==1.5.1 -attrs==26.1.0 -betterproto==2.0.0b7 -bidict==0.22.0 -boto3==1.40.53 -botocore==1.40.53 -cached_property==1.5.2 -cachetools==6.2.6 -certifi==2026.4.22 -charset_normalizer==3.4.7 -click==8.3.3 -Deprecated==1.2.14 -frozenlist==1.8.0 -fs==2.4.16 -fsspec==2025.9.0 -grpclib==0.4.9 -h2==4.3.0 -hpack==4.1.0 -hyperframe==6.1.0 -idna==3.15 -iniconfig==1.1.1 -jmespath==1.1.0 -loguru==0.7.0 -markdown-it-py==4.2.0 -mdurl==0.1.2 -mmh3==5.2.1 -multidict==6.7.1 -numpy==2.1.0 -overrides==7.4.0 -packaging==26.2 -pampy==0.3.0 -pandas==2.2.3 -pg8000==1.31.5 -pluggy==1.6.0 -praw==7.6.1 -prawcore==2.4.0 -propcache==0.5.2 -protobuf==7.34.1 -psutil==5.9.0 -pyarrow==21.0.0 -pydantic==2.13.4 -pydantic-core==2.46.4 -pygments==2.20.0 -pyiceberg==0.11.1 -pympler==1.1 -pyparsing==3.3.2 -pyroaring==1.1.0 -pytest==7.4.0 -pytest-reraise==2.1.2 -pytest-timeout==2.2.0 -python-dateutil==2.8.2 -pytz==2026.2 -readerwriterlock==1.0.9 -requests==2.34.0 -rich==14.3.4 -ruff==0.14.7 -s3fs==2025.9.0 -s3transfer==0.14.0 -scramp==1.4.8 -setuptools==80.10.2 -six==1.17.0 -SQLAlchemy==2.0.37 -strictyaml==1.7.3 -tenacity==8.5.0 -typing-inspection==0.4.2 -typing_extensions==4.14.1 -tzdata==2026.2 -tzlocal==2.1 -update-checker==0.18.0 -urllib3==2.7.0 -websocket-client==1.9.0 -wrapt==1.17.3 -yarl==1.23.0 -zstandard==0.25.0 \ No newline at end of file From 983b098c98e24d7b02d2f0fae45d6506c853e232 Mon Sep 17 00:00:00 2001 From: Sarah Asad Date: Wed, 10 Jun 2026 16:54:41 -0700 Subject: [PATCH 02/13] add spinner --- .../computing-unit-selection.component.html | 48 ++++++++++++------- .../computing-unit-selection.component.ts | 7 +++ frontend/src/styles.scss | 9 ++++ 3 files changed, 46 insertions(+), 18 deletions(-) diff --git a/frontend/src/app/workspace/component/power-button/computing-unit-selection.component.html b/frontend/src/app/workspace/component/power-button/computing-unit-selection.component.html index 6167d367591..7fe7dca6777 100644 --- a/frontend/src/app/workspace/component/power-button/computing-unit-selection.component.html +++ b/frontend/src/app/workspace/component/power-button/computing-unit-selection.component.html @@ -472,28 +472,40 @@
-
-
Package
-
Version
+
+ + Fetching system packages…
-
-
- + +
+
Package
+
Version
+
- +
+
+ + + +
-
+
diff --git a/frontend/src/app/workspace/component/power-button/computing-unit-selection.component.ts b/frontend/src/app/workspace/component/power-button/computing-unit-selection.component.ts index b9a41c5b5f6..cc505cd38eb 100644 --- a/frontend/src/app/workspace/component/power-button/computing-unit-selection.component.ts +++ b/frontend/src/app/workspace/component/power-button/computing-unit-selection.component.ts @@ -139,6 +139,10 @@ export class ComputingUnitSelectionComponent implements OnInit { // variables for creating a virtual environment pves: PveDraft[] = []; systemPackages: { name: string; version: string }[] = []; + // True while the first /pve/system response is in flight. The server resolves + // the full pinned set with a one-time `pip freeze` against a throwaway venv, + // which can take 30–60s on the first request after a server restart. + systemPackagesLoading = false; pveModalVisible = false; // current workflow's Id, will change with wid in the workflowActionService.metadata @@ -800,6 +804,7 @@ export class ComputingUnitSelectionComponent implements OnInit { isLocked: true, })); + this.systemPackagesLoading = true; this.workflowPveService .getSystemPackages(cuId) .pipe(untilDestroyed(this)) @@ -812,10 +817,12 @@ export class ComputingUnitSelectionComponent implements OnInit { version: (version ?? "").trim(), }; }); + this.systemPackagesLoading = false; }, error: (err: unknown) => { console.error("Failed to fetch system packages:", err); this.systemPackages = []; + this.systemPackagesLoading = false; }, }); }, diff --git a/frontend/src/styles.scss b/frontend/src/styles.scss index 5f4a00952a9..d761cf56a56 100644 --- a/frontend/src/styles.scss +++ b/frontend/src/styles.scss @@ -161,6 +161,15 @@ hr { gap: 6px; } + .system-loading { + display: flex; + align-items: center; + gap: 8px; + padding: 12px 4px; + color: rgba(0, 0, 0, 0.55); + font-style: italic; + } + .package-header-row, .package-inputs { display: grid; From b8229b2d8405f81081362b6f2ebf0e095021fe05 Mon Sep 17 00:00:00 2001 From: Sarah Asad Date: Wed, 10 Jun 2026 17:00:23 -0700 Subject: [PATCH 03/13] use tmp pve to get system packages --- .../pythonvirtualenvironment/PveManager.scala | 168 +++++++++++------- .../PveResource.scala | 9 +- .../PveResourceSpec.scala | 19 +- 3 files changed, 111 insertions(+), 85 deletions(-) diff --git a/amber/src/main/scala/org/apache/texera/web/resource/pythonvirtualenvironment/PveManager.scala b/amber/src/main/scala/org/apache/texera/web/resource/pythonvirtualenvironment/PveManager.scala index 2256798030b..cc80474867f 100644 --- a/amber/src/main/scala/org/apache/texera/web/resource/pythonvirtualenvironment/PveManager.scala +++ b/amber/src/main/scala/org/apache/texera/web/resource/pythonvirtualenvironment/PveManager.scala @@ -19,13 +19,15 @@ package org.apache.texera.web.resource.pythonvirtualenvironment +import com.typesafe.scalalogging.LazyLogging +import org.apache.texera.amber.config.PythonUtils + import java.nio.file.{Files, Path, Paths} +import java.util.Comparator import java.util.concurrent.BlockingQueue import scala.collection.mutable.Map import scala.jdk.CollectionConverters._ import scala.sys.process._ -import java.util.Comparator -import org.apache.texera.amber.config.PythonUtils /** * PveManager is responsible for managing Python Virtual Environments (PVEs) @@ -40,7 +42,7 @@ import org.apache.texera.amber.config.PythonUtils * /tmp/texera-pve/venvs/{cuid}/{pveName}/ */ -object PveManager { +object PveManager extends LazyLogging { case class PvePackageResponse( pveName: String, @@ -94,26 +96,91 @@ object PveManager { } } - private def getSystemPath(isLocal: Boolean): Path = { - Paths.get( - if (isLocal) "amber/system-requirements-lock.txt" - else "/tmp/system-requirements-lock.txt" - ) - } + private def locateRequirementsTxt(): Option[Path] = + Seq(Paths.get("/tmp", "requirements.txt"), Paths.get("amber", "requirements.txt")) + .find(Files.exists(_)) + + // Resolves the fully-pinned system package set by installing requirements.txt + // into a throwaway venv and running `pip freeze`. + private def resolveSystemPackages(): Seq[String] = { + val requirementsPath = locateRequirementsTxt() match { + case Some(p) => p + case None => + logger.error("requirements.txt not found; system package set will be empty") + return Seq.empty + } - def getSystemPackages(isLocal: Boolean): Seq[String] = { - if (!Files.exists(getSystemPath(isLocal))) { - Seq() - } else { - Files - .readAllLines(getSystemPath(isLocal)) - .asScala - .map(_.trim) - .filter(line => line.nonEmpty && !line.startsWith("#")) - .toSeq + val tempVenv = Files.createTempDirectory("texera-system-venv-") + try { + val python = tempVenv.resolve("bin").resolve("python").toString + val createCode = + Process(Seq(PythonUtils.getPythonExecutable, "-m", "venv", tempVenv.toString)).! + if (createCode != 0) { + logger.error(s"failed to create temp venv for system-package resolution (exit=$createCode)") + return Seq.empty + } + + val installCode = Process( + Seq( + python, + "-u", + "-m", + "pip", + "install", + "--progress-bar", + "off", + "--no-input", + "-r", + requirementsPath.toString + ), + None, + pipEnv.toSeq: _* + ).! + if (installCode != 0) { + logger.error(s"failed to install requirements into temp venv (exit=$installCode)") + return Seq.empty + } + + val collected = scala.collection.mutable.ListBuffer[String]() + val freezeCode = Process(Seq(python, "-m", "pip", "freeze")).!( + ProcessLogger(line => collected += line, _ => ()) + ) + if (freezeCode != 0) { + logger.error(s"pip freeze failed (exit=$freezeCode)") + return Seq.empty + } + collected.toSeq.map(_.trim).filter(line => line.nonEmpty && !line.startsWith("#")) + } finally { + try { + val stream = Files.walk(tempVenv) + try stream.sorted(Comparator.reverseOrder()).iterator().asScala.foreach(Files.deleteIfExists) + finally stream.close() + } catch { + case _: Throwable => () // best-effort cleanup + } } } + // Cached for the JVM lifetime. The system Python + requirements.txt don't + // change without an app restart, so resolving once is sufficient. + private lazy val systemPackages: Seq[String] = resolveSystemPackages() + + // Normalised package names ("numpy", "pandas") — used to reject user + // attempts to install or delete system packages. + private lazy val systemPackageNames: Set[String] = + systemPackages.map(_.split("==")(0).trim.toLowerCase).toSet + + // Materialised once: a file containing the frozen system requirements, + // passed as `pip install --constraint` so user installs respect system pins. + private lazy val systemConstraintFile: Path = { + val f = Files.createTempFile("texera-system-constraint-", ".txt") + Files.write(f, systemPackages.asJava) + f.toFile.deleteOnExit() + f + } + + def getSystemPackages: Seq[String] = systemPackages + private def runPipInstall( python: String, args: Seq[String], @@ -153,20 +220,15 @@ object PveManager { def createNewPve( cuid: Int, queue: BlockingQueue[String], - pveName: String, - isLocal: Boolean + pveName: String ): Unit = { queue.put(s"[PVE] Creating new PVE for cuid: $cuid with name: $pveName") - // NOTE: These paths are derived from computing-unit-master.dockerfile. - // If requirements.txt location changes, update these paths. - val requirementsPath = - if (isLocal) Paths.get("amber", "requirements.txt") - else Paths.get("/tmp", "requirements.txt") - - if (!Files.exists(requirementsPath)) { - queue.put(s"[PVE][ERR] System requirements not found") - return + val requirementsPath = locateRequirementsTxt() match { + case Some(p) => p + case None => + queue.put(s"[PVE][ERR] System requirements not found") + return } val venvDirPath = pveDir(cuid, pveName).toAbsolutePath @@ -279,8 +341,7 @@ object PveManager { packages: List[String], cuid: Int, queue: BlockingQueue[String], - pveName: String, - isLocal: Boolean + pveName: String ): Unit = { val python = pythonBinPath(cuid, pveName).toAbsolutePath.toString @@ -294,19 +355,6 @@ object PveManager { var installedPackages = readPackageFile(metadataPath).toSet - val systemPackages = - if (Files.exists(getSystemPath(isLocal))) { - Files - .readAllLines(getSystemPath(isLocal)) - .asScala - .map(_.trim) - .filter(line => line.nonEmpty && !line.startsWith("#")) - .map(line => line.split("==")(0).trim.toLowerCase) - .toSet - } else { - Set[String]() - } - packages.foreach { pkg => val trimmedPkg = pkg.trim @@ -314,7 +362,7 @@ object PveManager { val userPackageName = trimmedPkg.split("==")(0).trim.toLowerCase - if (systemPackages.contains(userPackageName)) { + if (systemPackageNames.contains(userPackageName)) { queue.put( s"[PVE][ERR] $trimmedPkg is a system package and cannot be installed or modified by the user." ) @@ -326,8 +374,8 @@ object PveManager { val code = runPipInstall( python, Seq( - "--constraint", // check against system-requirements-lock - getSystemPath(isLocal).toString, + "--constraint", // pin to the runtime-resolved system set + systemConstraintFile.toString, trimmedPkg ), queue @@ -365,35 +413,21 @@ object PveManager { def deletePackages( cuid: Int, packageName: String, - pveName: String, - isLocal: Boolean + pveName: String ): List[String] = { val python = pythonBinPath(cuid, pveName).toAbsolutePath.toString val metadataPath = cuidDir(cuid, pveName).resolve("user-packages.txt") if (!Files.exists(Paths.get(python))) { val msg = s"[PVE][ERR] Python executable not found for PVE: $python" - println(msg) + logger.error(msg) return List(msg) } val trimmedPackageName = packageName.trim val normalizedPackageName = trimmedPackageName.split("==")(0).trim.toLowerCase - val systemPackages = - if (Files.exists(getSystemPath(isLocal))) { - Files - .readAllLines(getSystemPath(isLocal)) - .asScala - .map(_.trim) - .filter(line => line.nonEmpty && !line.startsWith("#")) - .map(line => line.split("==")(0).trim.toLowerCase) - .toSet - } else { - Set[String]() - } - - if (systemPackages.contains(normalizedPackageName)) { + if (systemPackageNames.contains(normalizedPackageName)) { return List( s"[PVE][ERR] $trimmedPackageName is a system package and cannot be deleted." ) @@ -419,11 +453,11 @@ object PveManager { val exitCode = command.!( ProcessLogger( out => { - println(s"[pip] $out") + logger.info(s"[pip] $out") output += s"[pip] $out" }, err => { - System.err.println(s"[pip][ERR] $err") + logger.error(s"[pip][ERR] $err") output += s"[pip][ERR] $err" } ) diff --git a/amber/src/main/scala/org/apache/texera/web/resource/pythonvirtualenvironment/PveResource.scala b/amber/src/main/scala/org/apache/texera/web/resource/pythonvirtualenvironment/PveResource.scala index ac07616d509..82e3519688a 100644 --- a/amber/src/main/scala/org/apache/texera/web/resource/pythonvirtualenvironment/PveResource.scala +++ b/amber/src/main/scala/org/apache/texera/web/resource/pythonvirtualenvironment/PveResource.scala @@ -19,8 +19,6 @@ package org.apache.texera.web.resource.pythonvirtualenvironment -import org.apache.texera.config.KubernetesConfig - import javax.ws.rs._ import javax.ws.rs.core.MediaType import scala.jdk.CollectionConverters._ @@ -39,10 +37,9 @@ class PveResource { @Path("/system") @Produces(Array(MediaType.APPLICATION_JSON)) def getSystemPackages: util.Map[String, util.List[String]] = { - val isLocal = !KubernetesConfig.kubernetesComputingUnitEnabled try { val systemPkgs = - PveManager.getSystemPackages(isLocal).toList.asJava + PveManager.getSystemPackages.toList.asJava Map("system" -> systemPkgs).asJava } catch { @@ -104,12 +101,10 @@ class PveResource { @PathParam("pveName") pveName: String, @PathParam("packageName") packageName: String ): Response = { - val isLocal = !KubernetesConfig.kubernetesComputingUnitEnabled val messages = PveManager.deletePackages( cuid, packageName, - pveName, - isLocal + pveName ) if (messages.exists(_.contains("[PVE][ERR]"))) { diff --git a/amber/src/test/scala/org/apache/texera/web/resource/pythonvirtualenvironment/PveResourceSpec.scala b/amber/src/test/scala/org/apache/texera/web/resource/pythonvirtualenvironment/PveResourceSpec.scala index 92e83615c9f..15d66063be1 100644 --- a/amber/src/test/scala/org/apache/texera/web/resource/pythonvirtualenvironment/PveResourceSpec.scala +++ b/amber/src/test/scala/org/apache/texera/web/resource/pythonvirtualenvironment/PveResourceSpec.scala @@ -49,7 +49,7 @@ class PveResourceSpec extends AnyFlatSpec with Matchers with BeforeAndAfterEach } "PveManager" should "create a new PVE and list it" in { - PveManager.createNewPve(testCuid, queue, testPveName, isLocal = true) + PveManager.createNewPve(testCuid, queue, testPveName) val logs = queueText() @@ -68,7 +68,7 @@ class PveResourceSpec extends AnyFlatSpec with Matchers with BeforeAndAfterEach } "PveManager" should "install a user package and list it for the PVE" in { - PveManager.createNewPve(testCuid, queue, testPveName, isLocal = true) + PveManager.createNewPve(testCuid, queue, testPveName) val packageName = "colorama" val packageVersion = "0.4.6" @@ -80,8 +80,7 @@ class PveResourceSpec extends AnyFlatSpec with Matchers with BeforeAndAfterEach List(packageSpec), testCuid, queue, - testPveName, - isLocal = true + testPveName ) val logs = queueText() @@ -99,7 +98,7 @@ class PveResourceSpec extends AnyFlatSpec with Matchers with BeforeAndAfterEach } "PveManager" should "delete a user package and remove it from the PVE package list" in { - PveManager.createNewPve(testCuid, queue, testPveName, isLocal = true) + PveManager.createNewPve(testCuid, queue, testPveName) val packageName = "colorama" val packageVersion = "0.4.6" @@ -111,8 +110,7 @@ class PveResourceSpec extends AnyFlatSpec with Matchers with BeforeAndAfterEach List(packageSpec), testCuid, queue, - testPveName, - isLocal = true + testPveName ) PveManager @@ -124,8 +122,7 @@ class PveResourceSpec extends AnyFlatSpec with Matchers with BeforeAndAfterEach val deleteLogs = PveManager.deletePackages( testCuid, packageName, - testPveName, - isLocal = true + testPveName ) deleteLogs.mkString("\n") should not include "[PVE][ERR]" @@ -140,7 +137,7 @@ class PveResourceSpec extends AnyFlatSpec with Matchers with BeforeAndAfterEach } "PveManager" should "delete all PVEs for a computing unit" in { - PveManager.createNewPve(testCuid, queue, testPveName, isLocal = true) + PveManager.createNewPve(testCuid, queue, testPveName) Files.exists(testRoot.resolve(testPveName)) shouldBe true @@ -151,7 +148,7 @@ class PveResourceSpec extends AnyFlatSpec with Matchers with BeforeAndAfterEach } "PveManager.getPythonBin" should "return Some for an existing venv" in { - PveManager.createNewPve(testCuid, queue, testPveName, isLocal = true) + PveManager.createNewPve(testCuid, queue, testPveName) val result = PveManager.getPythonBin(testCuid, testPveName) result shouldBe defined From b904b3c6c71df43bb3679b7db3ecf6b33e535ab8 Mon Sep 17 00:00:00 2001 From: Sarah Asad Date: Wed, 10 Jun 2026 17:00:28 -0700 Subject: [PATCH 04/13] update test --- .../pythonvirtualenvironment/PveWebsocketResource.scala | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/amber/src/main/scala/org/apache/texera/web/resource/pythonvirtualenvironment/PveWebsocketResource.scala b/amber/src/main/scala/org/apache/texera/web/resource/pythonvirtualenvironment/PveWebsocketResource.scala index efaa266caac..34654c7604c 100644 --- a/amber/src/main/scala/org/apache/texera/web/resource/pythonvirtualenvironment/PveWebsocketResource.scala +++ b/amber/src/main/scala/org/apache/texera/web/resource/pythonvirtualenvironment/PveWebsocketResource.scala @@ -19,8 +19,6 @@ package org.apache.texera.web.resource.pythonvirtualenvironment -import org.apache.texera.config.KubernetesConfig - import javax.websocket._ import javax.websocket.server.ServerEndpoint import java.util.concurrent.LinkedBlockingQueue @@ -43,7 +41,6 @@ class PveWebsocketResource { val cuid = params.get("cuid").get(0).toInt val pveName = params.get("pveName").get(0) - val isLocal = !KubernetesConfig.kubernetesComputingUnitEnabled val action = params.getOrDefault("action", java.util.List.of("create")).get(0) val queue = new LinkedBlockingQueue[String]() @@ -52,7 +49,7 @@ class PveWebsocketResource { try { action match { case "create" => - PveManager.createNewPve(cuid, queue, pveName, isLocal) + PveManager.createNewPve(cuid, queue, pveName) case "install" => val packages = @@ -66,7 +63,7 @@ class PveWebsocketResource { .map(_.replace("\"", "").trim) .filter(_.nonEmpty) - PveManager.installUserPackages(packages, cuid, queue, pveName, isLocal) + PveManager.installUserPackages(packages, cuid, queue, pveName) case _ => queue.put(s"[ERR] Unknown action: $action") From 727a936e4e65d274003b87a50f30e76e917e0ede Mon Sep 17 00:00:00 2001 From: Sarah Asad Date: Wed, 10 Jun 2026 17:13:02 -0700 Subject: [PATCH 05/13] formatting --- .../web/resource/pythonvirtualenvironment/PveManager.scala | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/amber/src/main/scala/org/apache/texera/web/resource/pythonvirtualenvironment/PveManager.scala b/amber/src/main/scala/org/apache/texera/web/resource/pythonvirtualenvironment/PveManager.scala index cc80474867f..5089e68be6d 100644 --- a/amber/src/main/scala/org/apache/texera/web/resource/pythonvirtualenvironment/PveManager.scala +++ b/amber/src/main/scala/org/apache/texera/web/resource/pythonvirtualenvironment/PveManager.scala @@ -153,7 +153,11 @@ object PveManager extends LazyLogging { } finally { try { val stream = Files.walk(tempVenv) - try stream.sorted(Comparator.reverseOrder()).iterator().asScala.foreach(Files.deleteIfExists) + try stream + .sorted(Comparator.reverseOrder()) + .iterator() + .asScala + .foreach(Files.deleteIfExists) finally stream.close() } catch { case _: Throwable => () // best-effort cleanup From d0dbd22dbc3060f7f47496c4bfa3660832433c88 Mon Sep 17 00:00:00 2001 From: Sarah Asad Date: Fri, 12 Jun 2026 00:00:01 -0700 Subject: [PATCH 06/13] chore: move spinner UI to system-package-ui branch --- .../computing-unit-selection.component.html | 48 +++++++------------ .../computing-unit-selection.component.ts | 7 --- frontend/src/styles.scss | 9 ---- 3 files changed, 18 insertions(+), 46 deletions(-) diff --git a/frontend/src/app/workspace/component/power-button/computing-unit-selection.component.html b/frontend/src/app/workspace/component/power-button/computing-unit-selection.component.html index 7fe7dca6777..6167d367591 100644 --- a/frontend/src/app/workspace/component/power-button/computing-unit-selection.component.html +++ b/frontend/src/app/workspace/component/power-button/computing-unit-selection.component.html @@ -472,40 +472,28 @@
-
- - Fetching system packages… +
+
Package
+
Version
- -
-
Package
-
Version
-
- -
-
- +
+
+ - -
+
- +
diff --git a/frontend/src/app/workspace/component/power-button/computing-unit-selection.component.ts b/frontend/src/app/workspace/component/power-button/computing-unit-selection.component.ts index cc505cd38eb..b9a41c5b5f6 100644 --- a/frontend/src/app/workspace/component/power-button/computing-unit-selection.component.ts +++ b/frontend/src/app/workspace/component/power-button/computing-unit-selection.component.ts @@ -139,10 +139,6 @@ export class ComputingUnitSelectionComponent implements OnInit { // variables for creating a virtual environment pves: PveDraft[] = []; systemPackages: { name: string; version: string }[] = []; - // True while the first /pve/system response is in flight. The server resolves - // the full pinned set with a one-time `pip freeze` against a throwaway venv, - // which can take 30–60s on the first request after a server restart. - systemPackagesLoading = false; pveModalVisible = false; // current workflow's Id, will change with wid in the workflowActionService.metadata @@ -804,7 +800,6 @@ export class ComputingUnitSelectionComponent implements OnInit { isLocked: true, })); - this.systemPackagesLoading = true; this.workflowPveService .getSystemPackages(cuId) .pipe(untilDestroyed(this)) @@ -817,12 +812,10 @@ export class ComputingUnitSelectionComponent implements OnInit { version: (version ?? "").trim(), }; }); - this.systemPackagesLoading = false; }, error: (err: unknown) => { console.error("Failed to fetch system packages:", err); this.systemPackages = []; - this.systemPackagesLoading = false; }, }); }, diff --git a/frontend/src/styles.scss b/frontend/src/styles.scss index d761cf56a56..5f4a00952a9 100644 --- a/frontend/src/styles.scss +++ b/frontend/src/styles.scss @@ -161,15 +161,6 @@ hr { gap: 6px; } - .system-loading { - display: flex; - align-items: center; - gap: 8px; - padding: 12px 4px; - color: rgba(0, 0, 0, 0.55); - font-style: italic; - } - .package-header-row, .package-inputs { display: grid; From f5b98007a00c01eed91503142618fa04a51f3ba0 Mon Sep 17 00:00:00 2001 From: Sarah Asad Date: Fri, 12 Jun 2026 00:16:43 -0700 Subject: [PATCH 07/13] OS check --- .../pythonvirtualenvironment/PveManager.scala | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/amber/src/main/scala/org/apache/texera/web/resource/pythonvirtualenvironment/PveManager.scala b/amber/src/main/scala/org/apache/texera/web/resource/pythonvirtualenvironment/PveManager.scala index 5089e68be6d..93b9f61b806 100644 --- a/amber/src/main/scala/org/apache/texera/web/resource/pythonvirtualenvironment/PveManager.scala +++ b/amber/src/main/scala/org/apache/texera/web/resource/pythonvirtualenvironment/PveManager.scala @@ -20,6 +20,7 @@ package org.apache.texera.web.resource.pythonvirtualenvironment import com.typesafe.scalalogging.LazyLogging +import org.apache.commons.lang3.SystemUtils import org.apache.texera.amber.config.PythonUtils import java.nio.file.{Files, Path, Paths} @@ -60,8 +61,16 @@ object PveManager extends LazyLogging { private def pveDir(cuid: Int, pveName: String): Path = cuidDir(cuid, pveName).resolve("pve") + // Resolves the Python interpreter inside a venv. POSIX puts it at + // `/bin/python`; Windows puts it at `/Scripts/python.exe`. + private def venvPython(venvDir: Path): Path = + if (SystemUtils.IS_OS_WINDOWS) + venvDir.resolve("Scripts").resolve("python.exe") + else + venvDir.resolve("bin").resolve("python") + private def pythonBinPath(cuid: Int, pveName: String): Path = - pveDir(cuid, pveName).resolve("bin").resolve("python") + venvPython(pveDir(cuid, pveName)) /* * Validates the PVE name and returns the Python binary path if it exists, @@ -112,7 +121,7 @@ object PveManager extends LazyLogging { val tempVenv = Files.createTempDirectory("texera-system-venv-") try { - val python = tempVenv.resolve("bin").resolve("python").toString + val python = venvPython(tempVenv).toString val createCode = Process(Seq(PythonUtils.getPythonExecutable, "-m", "venv", tempVenv.toString)).! if (createCode != 0) { From 1ddc2ad55398f27bffa605b657a60a672e0b9f54 Mon Sep 17 00:00:00 2001 From: Sarah Asad Date: Fri, 12 Jun 2026 00:19:35 -0700 Subject: [PATCH 08/13] formatting --- .../web/resource/pythonvirtualenvironment/PveManager.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/amber/src/main/scala/org/apache/texera/web/resource/pythonvirtualenvironment/PveManager.scala b/amber/src/main/scala/org/apache/texera/web/resource/pythonvirtualenvironment/PveManager.scala index 93b9f61b806..20e94af2d4d 100644 --- a/amber/src/main/scala/org/apache/texera/web/resource/pythonvirtualenvironment/PveManager.scala +++ b/amber/src/main/scala/org/apache/texera/web/resource/pythonvirtualenvironment/PveManager.scala @@ -169,7 +169,7 @@ object PveManager extends LazyLogging { .foreach(Files.deleteIfExists) finally stream.close() } catch { - case _: Throwable => () // best-effort cleanup + case _: Throwable => () } } } From c8ead09fda07a82e1711010a2a28abcf819d1c60 Mon Sep 17 00:00:00 2001 From: Sarah Asad Date: Fri, 12 Jun 2026 16:37:55 -0700 Subject: [PATCH 09/13] formatting --- .../web/resource/pythonvirtualenvironment/PveManager.scala | 2 -- .../web/resource/pythonvirtualenvironment/PveResource.scala | 1 - 2 files changed, 3 deletions(-) diff --git a/amber/src/main/scala/org/apache/texera/web/resource/pythonvirtualenvironment/PveManager.scala b/amber/src/main/scala/org/apache/texera/web/resource/pythonvirtualenvironment/PveManager.scala index 69368d5d70b..0867b11ec79 100644 --- a/amber/src/main/scala/org/apache/texera/web/resource/pythonvirtualenvironment/PveManager.scala +++ b/amber/src/main/scala/org/apache/texera/web/resource/pythonvirtualenvironment/PveManager.scala @@ -21,10 +21,8 @@ package org.apache.texera.web.resource.pythonvirtualenvironment import com.typesafe.scalalogging.LazyLogging import org.apache.commons.lang3.SystemUtils -import org.apache.texera.amber.config.PythonUtils import java.nio.file.{Files, Path, Paths} -import java.util.Comparator import java.util.concurrent.BlockingQueue import scala.collection.mutable.Map import scala.jdk.CollectionConverters._ diff --git a/amber/src/main/scala/org/apache/texera/web/resource/pythonvirtualenvironment/PveResource.scala b/amber/src/main/scala/org/apache/texera/web/resource/pythonvirtualenvironment/PveResource.scala index 0ad874856b5..aeac02b1670 100644 --- a/amber/src/main/scala/org/apache/texera/web/resource/pythonvirtualenvironment/PveResource.scala +++ b/amber/src/main/scala/org/apache/texera/web/resource/pythonvirtualenvironment/PveResource.scala @@ -25,7 +25,6 @@ import com.fasterxml.jackson.module.scala.DefaultScalaModule import com.typesafe.scalalogging.LazyLogging import io.dropwizard.auth.Auth import org.apache.texera.auth.SessionUser -import org.apache.texera.config.KubernetesConfig import org.jooq.exception.DataAccessException import javax.ws.rs._ From d6d110aa318556429ee3b4fed8e91f331b2eabcb Mon Sep 17 00:00:00 2001 From: Sarah Asad Date: Fri, 12 Jun 2026 18:43:39 -0700 Subject: [PATCH 10/13] remove system lock --- bin/computing-unit-master.dockerfile | 1 - 1 file changed, 1 deletion(-) diff --git a/bin/computing-unit-master.dockerfile b/bin/computing-unit-master.dockerfile index 12d026ee543..df9928323ce 100644 --- a/bin/computing-unit-master.dockerfile +++ b/bin/computing-unit-master.dockerfile @@ -93,7 +93,6 @@ WORKDIR /texera/amber COPY --from=build /texera/amber/requirements.txt /tmp/requirements.txt COPY --from=build /texera/amber/operator-requirements.txt /tmp/operator-requirements.txt -COPY --from=build /texera/amber/system-requirements-lock.txt /tmp/system-requirements-lock.txt # Install Python runtime dependencies RUN apt-get update && apt-get install -y \ From 178530f626d1712ca95384b381ba26f3ca5831a6 Mon Sep 17 00:00:00 2001 From: Sarah Asad Date: Mon, 15 Jun 2026 18:07:39 -0700 Subject: [PATCH 11/13] add imports --- .../web/resource/pythonvirtualenvironment/PveManager.scala | 2 ++ 1 file changed, 2 insertions(+) diff --git a/amber/src/main/scala/org/apache/texera/web/resource/pythonvirtualenvironment/PveManager.scala b/amber/src/main/scala/org/apache/texera/web/resource/pythonvirtualenvironment/PveManager.scala index 0867b11ec79..c7cd6c2072a 100644 --- a/amber/src/main/scala/org/apache/texera/web/resource/pythonvirtualenvironment/PveManager.scala +++ b/amber/src/main/scala/org/apache/texera/web/resource/pythonvirtualenvironment/PveManager.scala @@ -30,6 +30,8 @@ import scala.sys.process._ import java.util.Comparator import org.apache.texera.amber.config.PythonUtils import org.apache.texera.dao.SqlServer +import com.typesafe.scalalogging.LazyLogging +import org.apache.commons.lang3.SystemUtils import org.apache.texera.dao.jooq.generated.tables.daos.VirtualEnvironmentsDao import org.apache.texera.dao.jooq.generated.tables.pojos.VirtualEnvironments import org.jooq.JSONB From dd0a03a8b7841d6b1689c612fa2f901fbc000d39 Mon Sep 17 00:00:00 2001 From: Sarah Asad Date: Mon, 15 Jun 2026 18:19:03 -0700 Subject: [PATCH 12/13] formatting --- .../web/resource/pythonvirtualenvironment/PveManager.scala | 2 -- .../web/resource/pythonvirtualenvironment/PveResource.scala | 1 - .../pythonvirtualenvironment/PveWebsocketResource.scala | 1 - 3 files changed, 4 deletions(-) diff --git a/amber/src/main/scala/org/apache/texera/web/resource/pythonvirtualenvironment/PveManager.scala b/amber/src/main/scala/org/apache/texera/web/resource/pythonvirtualenvironment/PveManager.scala index c09b2b742c5..4454c185d40 100644 --- a/amber/src/main/scala/org/apache/texera/web/resource/pythonvirtualenvironment/PveManager.scala +++ b/amber/src/main/scala/org/apache/texera/web/resource/pythonvirtualenvironment/PveManager.scala @@ -19,8 +19,6 @@ package org.apache.texera.web.resource.pythonvirtualenvironment -import com.typesafe.scalalogging.LazyLogging -import org.apache.commons.lang3.SystemUtils import java.nio.file.{Files, Path, Paths} import java.util.concurrent.BlockingQueue diff --git a/amber/src/main/scala/org/apache/texera/web/resource/pythonvirtualenvironment/PveResource.scala b/amber/src/main/scala/org/apache/texera/web/resource/pythonvirtualenvironment/PveResource.scala index 0b3ccc02eba..aeac02b1670 100644 --- a/amber/src/main/scala/org/apache/texera/web/resource/pythonvirtualenvironment/PveResource.scala +++ b/amber/src/main/scala/org/apache/texera/web/resource/pythonvirtualenvironment/PveResource.scala @@ -25,7 +25,6 @@ import com.fasterxml.jackson.module.scala.DefaultScalaModule import com.typesafe.scalalogging.LazyLogging import io.dropwizard.auth.Auth import org.apache.texera.auth.SessionUser -import org.apache.texera.common.config.KubernetesConfig import org.jooq.exception.DataAccessException import javax.ws.rs._ diff --git a/amber/src/main/scala/org/apache/texera/web/resource/pythonvirtualenvironment/PveWebsocketResource.scala b/amber/src/main/scala/org/apache/texera/web/resource/pythonvirtualenvironment/PveWebsocketResource.scala index d331d45bf6c..4415d36111d 100644 --- a/amber/src/main/scala/org/apache/texera/web/resource/pythonvirtualenvironment/PveWebsocketResource.scala +++ b/amber/src/main/scala/org/apache/texera/web/resource/pythonvirtualenvironment/PveWebsocketResource.scala @@ -19,7 +19,6 @@ package org.apache.texera.web.resource.pythonvirtualenvironment -import org.apache.texera.common.config.KubernetesConfig import javax.websocket._ import javax.websocket.server.ServerEndpoint From d3b341ea4c8ca5f467fae415cd1448e68b829528 Mon Sep 17 00:00:00 2001 From: Sarah Asad Date: Mon, 15 Jun 2026 18:24:16 -0700 Subject: [PATCH 13/13] formatting --- .../web/resource/pythonvirtualenvironment/PveManager.scala | 1 - .../resource/pythonvirtualenvironment/PveWebsocketResource.scala | 1 - 2 files changed, 2 deletions(-) diff --git a/amber/src/main/scala/org/apache/texera/web/resource/pythonvirtualenvironment/PveManager.scala b/amber/src/main/scala/org/apache/texera/web/resource/pythonvirtualenvironment/PveManager.scala index 4454c185d40..423a5b4a584 100644 --- a/amber/src/main/scala/org/apache/texera/web/resource/pythonvirtualenvironment/PveManager.scala +++ b/amber/src/main/scala/org/apache/texera/web/resource/pythonvirtualenvironment/PveManager.scala @@ -19,7 +19,6 @@ package org.apache.texera.web.resource.pythonvirtualenvironment - import java.nio.file.{Files, Path, Paths} import java.util.concurrent.BlockingQueue import scala.collection.mutable.Map diff --git a/amber/src/main/scala/org/apache/texera/web/resource/pythonvirtualenvironment/PveWebsocketResource.scala b/amber/src/main/scala/org/apache/texera/web/resource/pythonvirtualenvironment/PveWebsocketResource.scala index 4415d36111d..34654c7604c 100644 --- a/amber/src/main/scala/org/apache/texera/web/resource/pythonvirtualenvironment/PveWebsocketResource.scala +++ b/amber/src/main/scala/org/apache/texera/web/resource/pythonvirtualenvironment/PveWebsocketResource.scala @@ -19,7 +19,6 @@ package org.apache.texera.web.resource.pythonvirtualenvironment - import javax.websocket._ import javax.websocket.server.ServerEndpoint import java.util.concurrent.LinkedBlockingQueue