diff --git a/org.knime.python3.arrow.tests/META-INF/MANIFEST.MF b/org.knime.python3.arrow.tests/META-INF/MANIFEST.MF index 36dc71524..31ac6010e 100644 --- a/org.knime.python3.arrow.tests/META-INF/MANIFEST.MF +++ b/org.knime.python3.arrow.tests/META-INF/MANIFEST.MF @@ -12,5 +12,6 @@ Require-Bundle: org.junit;bundle-version="[4.13.0,5.0.0)", org.apache.commons.lang3;bundle-version="[3.9.0,4.0.0)", org.knime.core.columnar;bundle-version="[5.6.0,6.0.0)", org.knime.core.data.columnar;bundle-version="[5.6.0,6.0.0)", - org.knime.python3.testing;bundle-version="[5.6.0,6.0.0)" + org.knime.python3.testing;bundle-version="[5.6.0,6.0.0)", + org.knime.externalprocessprovider;bundle-version="[5.11.0,6.0.0)" Automatic-Module-Name: org.knime.python3.arrow.tests diff --git a/org.knime.python3.arrow.tests/src/test/java/org/knime/python3/arrow/TestUtils.java b/org.knime.python3.arrow.tests/src/test/java/org/knime/python3/arrow/TestUtils.java index 81c5497d7..a5dfb57eb 100644 --- a/org.knime.python3.arrow.tests/src/test/java/org/knime/python3/arrow/TestUtils.java +++ b/org.knime.python3.arrow.tests/src/test/java/org/knime/python3/arrow/TestUtils.java @@ -53,9 +53,9 @@ import java.util.Collections; import java.util.List; +import org.knime.externalprocessprovider.ExternalProcessProvider; import org.knime.python3.DefaultPythonGateway; import org.knime.python3.Python3SourceDirectory; -import org.knime.python3.PythonCommand; import org.knime.python3.PythonDataSink; import org.knime.python3.PythonDataSource; import org.knime.python3.PythonEntryPoint; @@ -83,7 +83,7 @@ private TestUtils() { * @throws InterruptedException */ public static PythonGateway openPythonGateway() throws IOException, InterruptedException { - final PythonCommand command = Python3TestUtils.getPythonCommand(); + final ExternalProcessProvider command = Python3TestUtils.getPythonCommand(); final String launcherPath = Paths.get(System.getProperty("user.dir"), "src/test/python", "tests_launcher.py").toString(); final PythonPath pythonPath = (new PythonPathBuilder()) // diff --git a/org.knime.python3.arrow.types.tests/META-INF/MANIFEST.MF b/org.knime.python3.arrow.types.tests/META-INF/MANIFEST.MF index c98d5943f..7d6d3dc73 100644 --- a/org.knime.python3.arrow.types.tests/META-INF/MANIFEST.MF +++ b/org.knime.python3.arrow.types.tests/META-INF/MANIFEST.MF @@ -16,6 +16,7 @@ Require-Bundle: org.knime.core.table;bundle-version="[5.6.0,6.0.0)", org.knime.core.data.columnar;bundle-version="[5.6.0,6.0.0)", com.fasterxml.jackson.core.jackson-databind;bundle-version="[2.12.1,3.0.0)", org.knime.python3.types;bundle-version="[5.6.0,6.0.0)", - org.knime.python3.testing;bundle-version="[5.6.0,6.0.0)" + org.knime.python3.testing;bundle-version="[5.6.0,6.0.0)", + org.knime.externalprocessprovider;bundle-version="[5.11.0,6.0.0)" Automatic-Module-Name: org.knime.python3.arrow.types.tests Export-Package: org.knime.python3.arrow.types diff --git a/org.knime.python3.arrow.types.tests/src/test/java/org/knime/python3/arrow/types/KnimeArrowExtensionTypesTest.java b/org.knime.python3.arrow.types.tests/src/test/java/org/knime/python3/arrow/types/KnimeArrowExtensionTypesTest.java index 55706f90f..90aed17c6 100644 --- a/org.knime.python3.arrow.types.tests/src/test/java/org/knime/python3/arrow/types/KnimeArrowExtensionTypesTest.java +++ b/org.knime.python3.arrow.types.tests/src/test/java/org/knime/python3/arrow/types/KnimeArrowExtensionTypesTest.java @@ -164,6 +164,7 @@ import org.knime.core.table.schema.DataSpecs.DataSpecWithTraits; import org.knime.core.table.schema.DefaultColumnarSchema; import org.knime.core.table.schema.traits.LogicalTypeTrait; +import org.knime.externalprocessprovider.ExternalProcessProvider; import org.knime.filehandling.core.connections.FSCategory; import org.knime.filehandling.core.connections.FSLocation; import org.knime.filehandling.core.data.location.FSLocationValue; @@ -172,7 +173,6 @@ import org.knime.filehandling.core.data.location.cell.SimpleFSLocationCellFactory; import org.knime.python3.DefaultPythonGateway; import org.knime.python3.Python3SourceDirectory; -import org.knime.python3.PythonCommand; import org.knime.python3.PythonDataSink; import org.knime.python3.PythonDataSource; import org.knime.python3.PythonEntryPoint; @@ -893,7 +893,7 @@ interface TriConsumer { private static PythonGateway openPythonGateway(final Class entryPointClass, final String launcherModule, final PythonModule... modules) throws IOException, InterruptedException { - final PythonCommand command = Python3TestUtils.getPythonCommand(); + final ExternalProcessProvider command = Python3TestUtils.getPythonCommand(); final String launcherPath = Paths.get(System.getProperty("user.dir"), "src/test/python", launcherModule) .toString(); final PythonPathBuilder builder = PythonPath.builder()// diff --git a/org.knime.python3.nodes/META-INF/MANIFEST.MF b/org.knime.python3.nodes/META-INF/MANIFEST.MF index 4429ecb86..556be0697 100644 --- a/org.knime.python3.nodes/META-INF/MANIFEST.MF +++ b/org.knime.python3.nodes/META-INF/MANIFEST.MF @@ -36,7 +36,8 @@ Require-Bundle: org.knime.core;bundle-version="[5.10.0,6.0.0)", com.fasterxml.jackson.core.jackson-annotations;bundle-version="[2.13.2,3.0.0)", org.knime.workflowservices;bundle-version="[5.10.0,6.0.0)", org.apache.commons.lang3;bundle-version="[3.9.0,4.0.0)", - org.knime.gateway.impl;bundle-version="[5.10.0,6.0.0)" + org.knime.gateway.impl;bundle-version="[5.10.0,6.0.0)", + org.knime.externalprocessprovider;bundle-version="[5.11.0,6.0.0)" Automatic-Module-Name: org.knime.python3.nodes Eclipse-RegisterBuddy: org.knime.ext.py4j Eclipse-BundleShape: dir diff --git a/org.knime.python3.nodes/src/main/java/org/knime/python3/nodes/PythonExtensionPreferences.java b/org.knime.python3.nodes/src/main/java/org/knime/python3/nodes/PythonExtensionPreferences.java index f483668fb..0505a75c0 100644 --- a/org.knime.python3.nodes/src/main/java/org/knime/python3/nodes/PythonExtensionPreferences.java +++ b/org.knime.python3.nodes/src/main/java/org/knime/python3/nodes/PythonExtensionPreferences.java @@ -61,8 +61,8 @@ import org.knime.conda.prefs.CondaPreferences; import org.knime.core.node.NodeLogger; +import org.knime.externalprocessprovider.ExternalProcessProvider; import org.knime.python3.CondaPythonCommand; -import org.knime.python3.PythonCommand; import org.knime.python3.SimplePythonCommand; import org.yaml.snakeyaml.Yaml; @@ -91,7 +91,7 @@ static Stream getPathsToCustomExtensions() { .map(Optional::get); } - static Optional getCustomPythonCommand(final String extensionId) { + static Optional getCustomPythonCommand(final String extensionId) { return loadConfigs()// .filter(e -> extensionId.equals(e.m_id))// .findFirst()// @@ -307,7 +307,7 @@ Optional getSrcPath() { } } - Optional getCommand() { + Optional getCommand() { if (m_condaEnvPath != null) { if (m_pythonExecutable != null) { LOGGER.warnWithFormat("Both conda_env_path and python_executable are provided for extension '%s'." diff --git a/org.knime.python3.nodes/src/main/java/org/knime/python3/nodes/PythonNodeGatewayFactory.java b/org.knime.python3.nodes/src/main/java/org/knime/python3/nodes/PythonNodeGatewayFactory.java index 12c6a7724..2a614bdab 100644 --- a/org.knime.python3.nodes/src/main/java/org/knime/python3/nodes/PythonNodeGatewayFactory.java +++ b/org.knime.python3.nodes/src/main/java/org/knime/python3/nodes/PythonNodeGatewayFactory.java @@ -53,11 +53,11 @@ import java.util.Objects; import org.knime.conda.envbundling.environment.CondaEnvironmentRegistry; +import org.knime.externalprocessprovider.ExternalProcessProvider; import org.knime.python3.Activator; import org.knime.python3.BundledPythonCommand; import org.knime.python3.FreshPythonGatewayFactory; import org.knime.python3.Python3SourceDirectory; -import org.knime.python3.PythonCommand; import org.knime.python3.PythonEntryPointUtils; import org.knime.python3.PythonGateway; import org.knime.python3.PythonGatewayFactory; @@ -141,12 +141,12 @@ public PythonGateway create() throws IOException, InterruptedE return gateway; } - private static PythonCommand createCommand(final String extensionId, final String environmentName) { + private static ExternalProcessProvider createCommand(final String extensionId, final String environmentName) { return PythonExtensionPreferences.getCustomPythonCommand(extensionId)// .orElseGet(() -> getPythonCommandForEnvironment(environmentName)); } - private static PythonCommand getPythonCommandForEnvironment(final String environmentName) { + private static ExternalProcessProvider getPythonCommandForEnvironment(final String environmentName) { final var environment = CondaEnvironmentRegistry.getEnvironment(environmentName); if (environment == null) { throw new IllegalStateException("Conda environment '" + environmentName + "' not found. " diff --git a/org.knime.python3.scripting.nodes/META-INF/MANIFEST.MF b/org.knime.python3.scripting.nodes/META-INF/MANIFEST.MF index 686ad0f7b..b6ba827bb 100644 --- a/org.knime.python3.scripting.nodes/META-INF/MANIFEST.MF +++ b/org.knime.python3.scripting.nodes/META-INF/MANIFEST.MF @@ -23,8 +23,9 @@ Require-Bundle: org.knime.core;bundle-version="[5.11.0,6.0.0)", org.eclipse.ui;bundle-version="3.119.0", org.knime.conda;bundle-version="[5.11.0,6.0.0)", org.knime.conda.envbundling;bundle-version="[5.10.0,6.0.0)", - org.knime.core.ui;bundle-version="[5.11.0,6.0.0)", - org.knime.workbench.editor;bundle-version="[5.10.0,6.0.0)", + org.knime.pixi.port;bundle-version="[5.11.0,6.0.0)", + org.knime.core.ui;bundle-version="[5.10.0,6.0.0)", + org.knime.workbench.editor;bundle-version="[5.9.0,6.0.0)", org.apache.batik.util;bundle-version="[1.16.0,2.0.0)", org.apache.batik.dom;bundle-version="[1.16.0,2.0.0)", org.apache.batik.anim;bundle-version="[1.16.0,2.0.0)", @@ -38,7 +39,8 @@ Require-Bundle: org.knime.core;bundle-version="[5.11.0,6.0.0)", com.fasterxml.jackson.core.jackson-core;bundle-version="2.13.2", com.fasterxml.jackson.core.jackson-databind;bundle-version="2.13.2", org.eclipse.orbit.xml-apis-ext;bundle-version="[1.0.0,2.0.0)", - org.apache.commons.commons-io;bundle-version="[2.15.1,3.0.0)" + org.apache.commons.commons-io;bundle-version="[2.15.1,3.0.0)", + org.knime.externalprocessprovider;bundle-version="[5.11.0,6.0.0)" Automatic-Module-Name: org.knime.python3.scripting.nodes Export-Package: org.knime.python3.scripting.nodes.prefs Eclipse-RegisterBuddy: org.knime.ext.py4j diff --git a/org.knime.python3.scripting.nodes/src/main/java/org/knime/python3/scripting/nodes/AbstractPythonScriptingNodeModel.java b/org.knime.python3.scripting.nodes/src/main/java/org/knime/python3/scripting/nodes/AbstractPythonScriptingNodeModel.java index a8e7f34e7..251a86aa4 100644 --- a/org.knime.python3.scripting.nodes/src/main/java/org/knime/python3/scripting/nodes/AbstractPythonScriptingNodeModel.java +++ b/org.knime.python3.scripting.nodes/src/main/java/org/knime/python3/scripting/nodes/AbstractPythonScriptingNodeModel.java @@ -81,6 +81,7 @@ import org.knime.core.util.PathUtils; import org.knime.core.util.asynclose.AsynchronousCloseableTracker; import org.knime.core.webui.node.view.NodeView; +import org.knime.externalprocessprovider.ExternalProcessProvider; import org.knime.python2.PythonCommand; import org.knime.python2.PythonModuleSpec; import org.knime.python2.PythonVersion; @@ -381,14 +382,14 @@ private void pushNewFlowVariable(final FlowVariable variable) { } /** - * Wraps a {@link org.knime.python3.PythonCommand} into the legacy implementation for using it in a + * Wraps a {@link ExternalProcessProvider} into the legacy implementation for using it in a * {@link PythonKernelBackend}. */ private static final class LegacyPythonCommand implements PythonCommand { - private final org.knime.python3.PythonCommand m_pythonCommand; + private final ExternalProcessProvider m_pythonCommand; - private LegacyPythonCommand(final org.knime.python3.PythonCommand pythonCommand) { + private LegacyPythonCommand(final ExternalProcessProvider pythonCommand) { m_pythonCommand = pythonCommand; } diff --git a/org.knime.python3.scripting.nodes/src/main/java/org/knime/python3/scripting/nodes/PortsConfigurationUtils.java b/org.knime.python3.scripting.nodes/src/main/java/org/knime/python3/scripting/nodes/PortsConfigurationUtils.java index c4be4324e..329da5b9d 100644 --- a/org.knime.python3.scripting.nodes/src/main/java/org/knime/python3/scripting/nodes/PortsConfigurationUtils.java +++ b/org.knime.python3.scripting.nodes/src/main/java/org/knime/python3/scripting/nodes/PortsConfigurationUtils.java @@ -50,8 +50,10 @@ import org.knime.core.node.BufferedDataTable; import org.knime.core.node.context.ports.PortsConfiguration; +import org.knime.core.node.port.PortObject; import org.knime.core.node.port.PortType; import org.knime.core.node.port.image.ImagePortObject; +import org.knime.pixi.port.PythonEnvironmentPortObject; import org.knime.python2.port.PickledObjectFileStorePortObject; import org.knime.python2.ports.DataTableInputPort; import org.knime.python2.ports.DataTableOutputPort; @@ -61,6 +63,7 @@ import org.knime.python2.ports.PickledObjectInputPort; import org.knime.python2.ports.PickledObjectOutputPort; + /** * @author Marcel Wiedenmann, KNIME GmbH, Konstanz, Germany * @author Benjamin Wilhelm, KNIME GmbH, Berlin, Germany @@ -71,6 +74,27 @@ private PortsConfigurationUtils() { // Utility class } + /** + * Check if the ports configuration contains a Python environment port. + * + * @param config the ports configuration + * @return true if a Python environment port is present + */ + public static boolean hasPythonEnvironmentPort(final PortsConfiguration config) { + final PortType[] inTypes = config.getInputPorts(); + try { + // Check if any input port is a PythonEnvironmentPortObject + for (final PortType inType : inTypes) { + if (isPythonEnvironmentPort(inType)) { + return true; + } + } + } catch (NoClassDefFoundError e) { + // Python environment bundle is not available - this is fine since it's optional + } + return false; + } + /** * Extract the input ports from the given ports configuration. * @@ -81,9 +105,21 @@ public static InputPort[] createInputPorts(final PortsConfiguration config) { final PortType[] inTypes = config.getInputPorts(); int inTableIndex = 0; int inObjectIndex = 0; - final var inPorts = new InputPort[inTypes.length]; + // Count non-environment ports for the result array + int numNonEnvironmentPorts = 0; + for (final PortType inType : inTypes) { + if (!isPythonEnvironmentPort(inType)) { + numNonEnvironmentPorts++; + } + } + final var inPorts = new InputPort[numNonEnvironmentPorts]; + int portIndex = 0; for (int i = 0; i < inTypes.length; i++) { final PortType inType = inTypes[i]; + // Skip Python environment ports - they are not InputPorts in the traditional sense + if (isPythonEnvironmentPort(inType)) { + continue; + } final InputPort inPort; if (BufferedDataTable.TYPE.equals(inType)) { inPort = new DataTableInputPort("knio.input_tables[" + inTableIndex++ + "]"); @@ -92,11 +128,26 @@ public static InputPort[] createInputPorts(final PortsConfiguration config) { } else { throw new IllegalStateException("Unsupported input type: " + inType.getName()); } - inPorts[i] = inPort; + inPorts[portIndex++] = inPort; } return inPorts; } + /** + * Check if a port type is a Python environment port. + * + * @param inType the port type to check + * @return true if the port type is a Python environment port + */ + public static boolean isPythonEnvironmentPort(final PortType inType) { + try { + return inType.equals(PythonEnvironmentPortObject.TYPE) || inType.equals(PythonEnvironmentPortObject.TYPE_OPTIONAL); + } catch (NoClassDefFoundError e) { + // Python environment bundle is not available - this is fine since it's optional + return false; + } + } + /** * Extract the output ports from the given ports configuration. * @@ -135,4 +186,48 @@ public static OutputPort[] createOutputPorts(final PortsConfiguration config) { public static OutputPort createPickledObjectOutputPort(final int outObjectSuffix) { return new PickledObjectOutputPort("knio.output_objects[" + outObjectSuffix + "]"); } -} + + /** + * Extract the Python environment port object from the input port objects, if present. + * + * @param inObjects the input port objects + * @return the Python environment port object, or null if not present + * @throws IllegalArgumentException if a Python environment port is expected but not found + */ + public static PythonEnvironmentPortObject extractPythonEnvironmentPort(final PortObject[] inObjects) { + for (final PortObject inObject : inObjects) { + if (inObject instanceof PythonEnvironmentPortObject envPort) { + return envPort; + } + } + throw new IllegalArgumentException( + "Expected a Python environment port object in the input ports, but none was found."); + } + + /*public static void installPythonEnvironmentIfPresent(final PortsConfiguration config, final PortObject[] inObjects, + final ExecutionMonitor exec) throws IOException, CanceledExecutionException { + final PythonEnvironmentPortObject envPort = extractPythonEnvironmentPort(config, inObjects); + if (envPort != null) { + PythonEnvironmentPortObject.installPythonEnvironmentWithProgress(config, inObjects, exec); + } + }*/ + + /** + * Filter out the Python environment port from the input port objects array. + * The environment port is not a data port and should not be passed to the session. + * + * @param inObjects the input port objects + * @return the filtered array without the Python environment port + */ + public static PortObject[] filterEnvironmentPort(final PortObject[] inObjects) { + // Find and exclude the environment port + final PortObject[] filtered = new PortObject[inObjects.length - 1]; + int filteredIndex = 0; + for (int i = 0; i < inObjects.length; i++) { + if (!(inObjects[i] instanceof PythonEnvironmentPortObject)) { + filtered[filteredIndex++] = inObjects[i]; + } + } + return filtered; + } +} \ No newline at end of file diff --git a/org.knime.python3.scripting.nodes/src/main/java/org/knime/python3/scripting/nodes/prefs/CondaEnvironmentConfig.java b/org.knime.python3.scripting.nodes/src/main/java/org/knime/python3/scripting/nodes/prefs/CondaEnvironmentConfig.java index dcaa8fc27..bd92f7504 100644 --- a/org.knime.python3.scripting.nodes/src/main/java/org/knime/python3/scripting/nodes/prefs/CondaEnvironmentConfig.java +++ b/org.knime.python3.scripting.nodes/src/main/java/org/knime/python3/scripting/nodes/prefs/CondaEnvironmentConfig.java @@ -56,8 +56,8 @@ import org.knime.conda.prefs.CondaPreferences; import org.knime.core.node.NodeLogger; import org.knime.core.node.defaultnodesettings.SettingsModelString; +import org.knime.externalprocessprovider.ExternalProcessProvider; import org.knime.python3.CondaPythonCommand; -import org.knime.python3.PythonCommand; /** * Copied and modified from org.knime.python2.config. @@ -113,7 +113,7 @@ public ObservableValue getAvailableEnvironments() } @Override - public PythonCommand getPythonCommand() { + public ExternalProcessProvider getPythonCommand() { return new CondaPythonCommand(CondaPreferences.getCondaInstallationDirectory(), m_environmentDirectory.getStringValue()); } diff --git a/org.knime.python3.scripting.nodes/src/main/java/org/knime/python3/scripting/nodes/prefs/ManualEnvironmentConfig.java b/org.knime.python3.scripting.nodes/src/main/java/org/knime/python3/scripting/nodes/prefs/ManualEnvironmentConfig.java index f0fd961b1..e814dc2cc 100644 --- a/org.knime.python3.scripting.nodes/src/main/java/org/knime/python3/scripting/nodes/prefs/ManualEnvironmentConfig.java +++ b/org.knime.python3.scripting.nodes/src/main/java/org/knime/python3/scripting/nodes/prefs/ManualEnvironmentConfig.java @@ -49,7 +49,7 @@ package org.knime.python3.scripting.nodes.prefs; import org.knime.core.node.defaultnodesettings.SettingsModelString; -import org.knime.python3.PythonCommand; +import org.knime.externalprocessprovider.ExternalProcessProvider; import org.knime.python3.SimplePythonCommand; /** @@ -79,7 +79,7 @@ public SettingsModelString getExecutablePath() { } @Override - public PythonCommand getPythonCommand() { + public ExternalProcessProvider getPythonCommand() { return new SimplePythonCommand(m_pythonPath.getStringValue()); } diff --git a/org.knime.python3.scripting.nodes/src/main/java/org/knime/python3/scripting/nodes/prefs/Python3ScriptingPreferences.java b/org.knime.python3.scripting.nodes/src/main/java/org/knime/python3/scripting/nodes/prefs/Python3ScriptingPreferences.java index 046407ca6..139c1ace1 100644 --- a/org.knime.python3.scripting.nodes/src/main/java/org/knime/python3/scripting/nodes/prefs/Python3ScriptingPreferences.java +++ b/org.knime.python3.scripting.nodes/src/main/java/org/knime/python3/scripting/nodes/prefs/Python3ScriptingPreferences.java @@ -52,8 +52,8 @@ import org.eclipse.core.runtime.preferences.InstanceScope; import org.knime.conda.CondaEnvironmentIdentifier; import org.knime.conda.prefs.CondaPreferences; +import org.knime.externalprocessprovider.ExternalProcessProvider; import org.knime.python3.BundledPythonCommand; -import org.knime.python3.PythonCommand; /** * Convenience front-end of the preference-based configuration of the Python integration. @@ -110,7 +110,7 @@ public static PythonEnvironmentType getEnvironmentTypePreference() { /** * @return The currently selected default Python command. */ - public static PythonCommand getPythonCommandPreference() { + public static ExternalProcessProvider getPythonCommandPreference() { final var envType = getEnvironmentTypePreference(); PythonEnvironmentsConfig environmentsConfig; @@ -124,7 +124,7 @@ public static PythonCommand getPythonCommandPreference() { } /** - * @return The {@link PythonCommand} for the installed bundled environment. + * @return The {@link ExternalProcessProvider} for the installed bundled environment. */ public static BundledPythonCommand getBundledPythonCommand() { return getBundledCondaEnvironmentConfig().getPythonCommand(); diff --git a/org.knime.python3.scripting.nodes/src/main/java/org/knime/python3/scripting/nodes/prefs/PythonEnvironmentConfig.java b/org.knime.python3.scripting.nodes/src/main/java/org/knime/python3/scripting/nodes/prefs/PythonEnvironmentConfig.java index 6999dce74..ef945ed64 100644 --- a/org.knime.python3.scripting.nodes/src/main/java/org/knime/python3/scripting/nodes/prefs/PythonEnvironmentConfig.java +++ b/org.knime.python3.scripting.nodes/src/main/java/org/knime/python3/scripting/nodes/prefs/PythonEnvironmentConfig.java @@ -50,7 +50,7 @@ import org.knime.core.node.defaultnodesettings.SettingsModelBoolean; import org.knime.core.node.defaultnodesettings.SettingsModelString; -import org.knime.python3.PythonCommand; +import org.knime.externalprocessprovider.ExternalProcessProvider; /** * Copied from org.knime.python2.config. @@ -63,7 +63,7 @@ interface PythonEnvironmentConfig extends PythonConfig { /** * @return The command that executes Python in the Python environment configured by this instance. */ - PythonCommand getPythonCommand(); + ExternalProcessProvider getPythonCommand(); /** * @return If the Python environment configured by this instance is currently the default environment. Not meant for diff --git a/org.knime.python3.scripting.nodes/src/main/java/org/knime/python3/scripting/nodes/prefs/PythonKernelTester.java b/org.knime.python3.scripting.nodes/src/main/java/org/knime/python3/scripting/nodes/prefs/PythonKernelTester.java index fa1b261a4..e1276b936 100644 --- a/org.knime.python3.scripting.nodes/src/main/java/org/knime/python3/scripting/nodes/prefs/PythonKernelTester.java +++ b/org.knime.python3.scripting.nodes/src/main/java/org/knime/python3/scripting/nodes/prefs/PythonKernelTester.java @@ -66,7 +66,7 @@ import org.knime.core.node.NodeLogger; import org.knime.core.util.FileUtil; import org.knime.core.util.Pair; -import org.knime.python3.PythonCommand; +import org.knime.externalprocessprovider.ExternalProcessProvider; /** * Copied from org.knime.python2. @@ -94,7 +94,7 @@ final class PythonKernelTester { * Caches previous test results. Mapping from the Python command that was tested to a list of additional required * modules that were tested along the command and the test results of these combinations of command and modules. */ - private static final Map, PythonKernelTestResult>>> TEST_RESULTS = + private static final Map, PythonKernelTestResult>>> TEST_RESULTS = new ConcurrentHashMap<>(); private static String getPythonKernelTesterPath() throws IOException { @@ -119,7 +119,7 @@ private PythonKernelTester() { * @param force Force the test to be rerun again even if the same configuration was successfully tested before. * @return The results of the installation test. */ - public static PythonKernelTestResult testPython3Installation(final PythonCommand python3Command, + public static PythonKernelTestResult testPython3Installation(final ExternalProcessProvider python3Command, final Collection additionalRequiredModules, final boolean force) { return testPythonInstallation(python3Command, PYTHON_MAJOR_VERSION_3, PYTHON_MINIMUM_VERSION_3, additionalRequiredModules, Collections.emptyList(), force); @@ -128,7 +128,7 @@ public static PythonKernelTestResult testPython3Installation(final PythonCommand /** * @param minimumVersion May be {@code null} in the case where no minimum version is required. */ - private static synchronized PythonKernelTestResult testPythonInstallation(final PythonCommand pythonCommand, + private static synchronized PythonKernelTestResult testPythonInstallation(final ExternalProcessProvider pythonCommand, final String majorVersion, final String minimumVersion, final Collection additionalRequiredModules, final Collection additionalOptionalModules, final boolean force) { @@ -189,7 +189,7 @@ private static synchronized PythonKernelTestResult testPythonInstallation(final return testResults; } - private static PythonKernelTestResult getPreviousTestResultsIfApplicable(final PythonCommand pythonCommand, + private static PythonKernelTestResult getPreviousTestResultsIfApplicable(final ExternalProcessProvider pythonCommand, final Set additionalRequiredModules) { // If a previous, appropriate Python test already succeeded, we will not have to run it again and return the // old results here (except if we're forced to). @@ -209,7 +209,7 @@ private static PythonKernelTestResult getPreviousTestResultsIfApplicable(final P return null; } - private static Process runPythonKernelTester(final PythonCommand pythonCommand, final String majorVersion, + private static Process runPythonKernelTester(final ExternalProcessProvider pythonCommand, final String majorVersion, final String minimumVersion, final Collection additionalRequiredModules, final Collection additionalOptionalModules, final StringBuilder testLogger) throws IOException { diff --git a/org.knime.python3.scripting.nodes/src/main/java/org/knime/python3/scripting/nodes2/ExecutableSelectionUtils.java b/org.knime.python3.scripting.nodes/src/main/java/org/knime/python3/scripting/nodes2/ExecutableSelectionUtils.java index 8551fa4d4..f5461f666 100644 --- a/org.knime.python3.scripting.nodes/src/main/java/org/knime/python3/scripting/nodes2/ExecutableSelectionUtils.java +++ b/org.knime.python3.scripting.nodes/src/main/java/org/knime/python3/scripting/nodes2/ExecutableSelectionUtils.java @@ -59,8 +59,8 @@ import org.knime.conda.CondaEnvironmentPropagation.CondaEnvironmentType; import org.knime.conda.prefs.CondaPreferences; import org.knime.core.node.workflow.FlowObjectStack; +import org.knime.externalprocessprovider.ExternalProcessProvider; import org.knime.python3.CondaPythonCommand; -import org.knime.python3.PythonCommand; import org.knime.python3.SimplePythonCommand; import org.knime.python3.scripting.nodes.prefs.Python3ScriptingPreferences; import org.knime.python3.scripting.nodes2.PythonScriptingService.ExecutableOption; @@ -83,7 +83,7 @@ static Map getExecutableOptions(final FlowObjectStack } /** Get the PythonCommand from the selected option */ - static PythonCommand getPythonCommand(final ExecutableOption option) { + static ExternalProcessProvider getPythonCommand(final ExecutableOption option) { switch (option.type) { case CONDA_ENV_VAR: return commandForConda(option.condaEnvDir); @@ -106,7 +106,7 @@ static PythonCommand getPythonCommand(final ExecutableOption option) { } /** Get the PythonCommand from the given settings String */ - static PythonCommand getPythonCommand(final String commandString) { + static ExternalProcessProvider getPythonCommand(final String commandString) { if (commandString == null || EXEC_SELECTION_PREF_ID.equals(commandString)) { // Nothing configured -> Use preferences return commandForPreferences(); @@ -126,7 +126,7 @@ static boolean isPathToCondaEnv(final String commandString) { private static ExecutableOption getPreferenceOption() { return new ExecutableOption(getPreferenceOptionType(), EXEC_SELECTION_PREF_ID, - commandForPreferences().getPythonExecutablePath().toString(), null, null); + commandForPreferences().getExecutablePath().toString(), null, null); } private static ExecutableOptionType getPreferenceOptionType() { @@ -161,15 +161,15 @@ private static Stream getCondaFlowVariableOptions(final FlowOb } } - private static PythonCommand commandForConda(final String condaEnvDir) { + private static ExternalProcessProvider commandForConda(final String condaEnvDir) { return new CondaPythonCommand(CondaPreferences.getCondaInstallationDirectory(), condaEnvDir); } - private static PythonCommand commandForString(final String pythonExecutable) { + private static ExternalProcessProvider commandForString(final String pythonExecutable) { return new SimplePythonCommand(pythonExecutable); } - private static PythonCommand commandForPreferences() { + private static ExternalProcessProvider commandForPreferences() { return Python3ScriptingPreferences.getPythonCommandPreference(); } } diff --git a/org.knime.python3.scripting.nodes/src/main/java/org/knime/python3/scripting/nodes2/PythonIOUtils.java b/org.knime.python3.scripting.nodes/src/main/java/org/knime/python3/scripting/nodes2/PythonIOUtils.java index 06bcb9375..0e36eb36b 100644 --- a/org.knime.python3.scripting.nodes/src/main/java/org/knime/python3/scripting/nodes2/PythonIOUtils.java +++ b/org.knime.python3.scripting.nodes/src/main/java/org/knime/python3/scripting/nodes2/PythonIOUtils.java @@ -79,6 +79,7 @@ import org.knime.core.node.port.image.ImagePortObjectSpec; import org.knime.core.node.port.inactive.InactiveBranchPortObject; import org.knime.core.util.PathUtils; +import org.knime.pixi.port.PythonEnvironmentPortObject; import org.knime.python2.port.PickledObjectFile; import org.knime.python2.port.PickledObjectFileStorePortObject; import org.knime.python3.PythonDataSource; @@ -100,10 +101,12 @@ private PythonIOUtils() { /** * Create an array of Python data sources for the given input ports. The input ports can be either a - * {@link BufferedDataTable} or a {@link PickledObjectFileStorePortObject}. + * {@link BufferedDataTable}, a {@link PickledObjectFileStorePortObject}, or {@link PythonEnvironmentPortObject}. + * Note that {@link PythonEnvironmentPortObject}s are filtered out as they are only used for environment + * configuration and not passed to Python as data. * - * @param data a list of port objects. Only {@link BufferedDataTable} and {@link PickledObjectFileStorePortObject} - * are supported. + * @param data a list of port objects. Only {@link BufferedDataTable}, {@link PickledObjectFileStorePortObject}, and + * {@link PythonEnvironmentPortObject} are supported. * @param tableConverter a table converter that is used to convert the {@link BufferedDataTable}s to Python sources * @param exec for progress reporting and cancellation * @return an array of Python data sources @@ -121,11 +124,6 @@ static PythonDataSource[] createSources(final PortObject[] data, final PythonArr .filter(BufferedDataTable.class::isInstance) // .toArray(BufferedDataTable[]::new); - // Make sure that all ports are tables or pickled port objects - if (pickledPortObjects.length + tablePortObjects.length < data.length) { - throw new IllegalArgumentException("Unsupported port type connected. This is an implementation error."); - } - // Progress handling final var pickledProgressWeight = 1; final var tableProgressWeight = 3; diff --git a/org.knime.python3.scripting.nodes/src/main/java/org/knime/python3/scripting/nodes2/PythonScriptNodeModel.java b/org.knime.python3.scripting.nodes/src/main/java/org/knime/python3/scripting/nodes2/PythonScriptNodeModel.java index eb698fa5f..23dddcc2b 100644 --- a/org.knime.python3.scripting.nodes/src/main/java/org/knime/python3/scripting/nodes2/PythonScriptNodeModel.java +++ b/org.knime.python3.scripting.nodes/src/main/java/org/knime/python3/scripting/nodes2/PythonScriptNodeModel.java @@ -86,8 +86,9 @@ import org.knime.core.node.workflow.VariableTypeRegistry; import org.knime.core.util.asynclose.AsynchronousCloseableTracker; import org.knime.core.webui.node.dialog.scripting.ScriptingService.ConsoleText; -import org.knime.python3.PythonCommand; +import org.knime.externalprocessprovider.ExternalProcessProvider; import org.knime.python3.PythonProcessTerminatedException; +import org.knime.python3.scripting.nodes.PortsConfigurationUtils; import org.knime.python3.scripting.nodes2.ConsoleOutputUtils.ConsoleOutputStorage; import org.knime.python3.scripting.nodes2.PythonScriptingSession.ExecutionInfo; import org.knime.python3.scripting.nodes2.PythonScriptingSession.ExecutionStatus; @@ -177,23 +178,44 @@ protected PortObjectSpec[] configure(final PortObjectSpec[] inSpecs) throws Inva @Override protected PortObject[] execute(final PortObject[] inObjects, final ExecutionContext exec) throws IOException, InterruptedException, CanceledExecutionException, KNIMEException { - final PythonCommand pythonCommand = - ExecutableSelectionUtils.getPythonCommand(m_settings.getExecutableSelection()); + var pythonEnvironmentPort = PortsConfigurationUtils.extractPythonEnvironmentPort(inObjects); + + // Install Python environment early to avoid timeout issues during gateway connection + final ExecutionMonitor remainingProgress; + final ExternalProcessProvider pythonCommand; + if (m_ports.hasPythonEnvironmentPort()) { + pythonEnvironmentPort.installPythonEnvironment(exec.createSubProgress(0.2)); + pythonCommand = pythonEnvironmentPort.getPythonCommand(); + remainingProgress = exec.createSubProgress(0.8); + } else { + // No environment port, just use the configured command and use the whole progress for the execution + remainingProgress = exec; + pythonCommand = ExecutableSelectionUtils.getPythonCommand(m_settings.getExecutableSelection()); + } + m_consoleOutputStorage = null; final var consoleConsumer = ConsoleOutputUtils.createConsoleConsumer(); try (final var session = new PythonScriptingSession(pythonCommand, consoleConsumer, new ModelFileStoreHandlerSupplier())) { - exec.setProgress(0.0, "Setting up inputs"); - session.setupIO(inObjects, getAvailableFlowVariables(KNOWN_FLOW_VARIABLE_TYPES).values(), + // Filter out environment port from inObjects - it's not a data port + final PortObject[] dataPortObjects; + if (m_ports.hasPythonEnvironmentPort()) { + dataPortObjects = PortsConfigurationUtils.filterEnvironmentPort(inObjects); + } else { + dataPortObjects = inObjects; + } + + remainingProgress.setProgress(0.0, "Setting up inputs"); + session.setupIO(dataPortObjects, getAvailableFlowVariables(KNOWN_FLOW_VARIABLE_TYPES).values(), m_ports.getNumOutTables(), m_ports.getNumOutImages(), m_ports.getNumOutObjects(), m_hasView, - exec.createSubProgress(0.3)); - exec.setProgress(0.3, "Running script"); + remainingProgress.createSubProgress(0.3)); + remainingProgress.setProgress(0.3, "Running script"); runUserScript(session); - exec.setProgress(0.7, "Processing output"); + remainingProgress.setProgress(0.7, "Processing output"); final var outputs = session.getOutputs(exec.createSubExecutionContext(0.3)); final var flowVars = session.getFlowVariables(); addNewFlowVariables(flowVars); @@ -272,7 +294,7 @@ private void addNewFlowVariables(final Collection newVariables) { } } - @SuppressWarnings({"unchecked", "rawtypes"}) + @SuppressWarnings("unchecked") private void pushNewFlowVariable(final FlowVariable variable) { pushFlowVariable(variable.getName(), (VariableType)variable.getVariableType(), variable.getValue(variable.getVariableType())); diff --git a/org.knime.python3.scripting.nodes/src/main/java/org/knime/python3/scripting/nodes2/PythonScriptPortsConfiguration.java b/org.knime.python3.scripting.nodes/src/main/java/org/knime/python3/scripting/nodes2/PythonScriptPortsConfiguration.java index 2897d4b48..8522c3af7 100644 --- a/org.knime.python3.scripting.nodes/src/main/java/org/knime/python3/scripting/nodes2/PythonScriptPortsConfiguration.java +++ b/org.knime.python3.scripting.nodes/src/main/java/org/knime/python3/scripting/nodes2/PythonScriptPortsConfiguration.java @@ -56,6 +56,7 @@ import org.knime.core.node.port.image.ImagePortObject; import org.knime.core.node.workflow.NodeContext; import org.knime.python2.port.PickledObjectFileStorePortObject; +import org.knime.python3.scripting.nodes.PortsConfigurationUtils; /** * Information about the configured ports of a Python scripting node. @@ -79,6 +80,9 @@ public final class PythonScriptPortsConfiguration { /** Name of the object output port */ public static final String PORTGR_ID_OUT_OBJECT = "Output object (pickled)"; + /** Name of the Python environment port (accepts PythonEnvironmentPortObject) */ + public static final String PORTGR_ID_PYTHON_ENV = "Python environment"; + private final int m_numInTables; private final int m_numInObjects; @@ -91,24 +95,31 @@ public final class PythonScriptPortsConfiguration { private final boolean m_hasView; + private final boolean m_hasPythonEnvironmentPort; + /** * Create a new {@link PythonScriptPortsConfiguration} from the given {@link PortsConfiguration}. * * @param portsConfig * @return a new {@link PythonScriptPortsConfiguration} */ - static PythonScriptPortsConfiguration fromPortsConfiguration(final PortsConfiguration portsConfig, final boolean hasView) { + static PythonScriptPortsConfiguration fromPortsConfiguration(final PortsConfiguration portsConfig, + final boolean hasView) { // Get the number of different output ports from the ports configuration (ArrayUtils#getLength handles null) final Map inPortsLocation = portsConfig.getInputPortLocation(); final var numInTables = ArrayUtils.getLength(inPortsLocation.get(PORTGR_ID_INP_TABLE)); final var numInObjects = ArrayUtils.getLength(inPortsLocation.get(PORTGR_ID_INP_OBJECT)); + // Check for environment port (accepts PythonEnvironmentPortObject) + final var hasEnvironmentPort = inPortsLocation.containsKey(PORTGR_ID_PYTHON_ENV) + && ArrayUtils.getLength(inPortsLocation.get(PORTGR_ID_PYTHON_ENV)) > 0; final Map outPortsLocation = portsConfig.getOutputPortLocation(); final var numOutTables = ArrayUtils.getLength(outPortsLocation.get(PORTGR_ID_OUT_TABLE)); final var numOutImages = ArrayUtils.getLength(outPortsLocation.get(PORTGR_ID_OUT_IMAGE)); final var numOutObjects = ArrayUtils.getLength(outPortsLocation.get(PORTGR_ID_OUT_OBJECT)); - return new PythonScriptPortsConfiguration(numInTables, numInObjects, numOutTables, numOutImages, numOutObjects, hasView); + return new PythonScriptPortsConfiguration(numInTables, numInObjects, numOutTables, numOutImages, numOutObjects, + hasView, hasEnvironmentPort); } /** @@ -134,12 +145,15 @@ static PythonScriptPortsConfiguration fromCurrentNodeContext() { // Count the number of the different ports (skip the flow var port) var numInTables = 0; var numInObjects = 0; + var hasEnvironmentPort = false; for (int i = 1; i < nodeContainer.getNrInPorts(); i++) { var portType = nodeContainer.getInPort(i).getPortType(); if (BufferedDataTable.TYPE.equals(portType)) { numInTables++; } else if (PickledObjectFileStorePortObject.TYPE.equals(portType)) { numInObjects++; + } else if (PortsConfigurationUtils.isPythonEnvironmentPort(portType)) { + hasEnvironmentPort = true; } else { throw new IllegalStateException("Unsupported input port configured. This is an implementation error."); } @@ -162,17 +176,20 @@ static PythonScriptPortsConfiguration fromCurrentNodeContext() { } var hasView = nodeContainer.getNrViews() > 0; - return new PythonScriptPortsConfiguration(numInTables, numInObjects, numOutTables, numOutImages, numOutObjects, hasView); + return new PythonScriptPortsConfiguration(numInTables, numInObjects, numOutTables, numOutImages, numOutObjects, + hasView, hasEnvironmentPort); } private PythonScriptPortsConfiguration(final int numInTables, final int numInObjects, final int numOutTables, - final int numOutImages, final int numOutObjects, final boolean hasView) { + final int numOutImages, final int numOutObjects, final boolean hasView, + final boolean hasPythonEnvironmentPort) { m_numInTables = numInTables; m_numInObjects = numInObjects; m_numOutTables = numOutTables; m_numOutImages = numOutImages; m_numOutObjects = numOutObjects; m_hasView = hasView; + m_hasPythonEnvironmentPort = hasPythonEnvironmentPort; } /** @@ -217,4 +234,10 @@ public boolean hasView() { return m_hasView; } + /** + * @return if the node has a Python environment port (accepts PythonEnvironmentPortObject) + */ + public boolean hasPythonEnvironmentPort() { + return m_hasPythonEnvironmentPort; + } } diff --git a/org.knime.python3.scripting.nodes/src/main/java/org/knime/python3/scripting/nodes2/PythonScriptingInputOutputModelUtils.java b/org.knime.python3.scripting.nodes/src/main/java/org/knime/python3/scripting/nodes2/PythonScriptingInputOutputModelUtils.java index a194596a7..950f30700 100644 --- a/org.knime.python3.scripting.nodes/src/main/java/org/knime/python3/scripting/nodes2/PythonScriptingInputOutputModelUtils.java +++ b/org.knime.python3.scripting.nodes/src/main/java/org/knime/python3/scripting/nodes2/PythonScriptingInputOutputModelUtils.java @@ -64,6 +64,7 @@ import org.knime.core.node.workflow.FlowVariable; import org.knime.core.webui.node.dialog.scripting.InputOutputModel; import org.knime.core.webui.node.dialog.scripting.WorkflowControl.InputPortInfo; +import org.knime.pixi.port.PythonEnvironmentPortObject; import org.knime.python2.port.PickledObjectFileStorePortObject; /** @@ -149,6 +150,8 @@ static List getInputObjects(final InputPortInfo[] inputPorts) // Object (spec not used) inputInfos.add(createInputModel(objectIdx, INPUT_OUTPUT_TYPE_OBJECT)); objectIdx++; + } else if (type.acceptsPortObjectClass(PythonEnvironmentPortObject.class)) { + // Skip Python environment ports - they are not data ports } else { throw new IllegalStateException("Unsupported input port. This is an implementation error."); } diff --git a/org.knime.python3.scripting.nodes/src/main/java/org/knime/python3/scripting/nodes2/PythonScriptingService.java b/org.knime.python3.scripting.nodes/src/main/java/org/knime/python3/scripting/nodes2/PythonScriptingService.java index be15da380..40def317b 100644 --- a/org.knime.python3.scripting.nodes/src/main/java/org/knime/python3/scripting/nodes2/PythonScriptingService.java +++ b/org.knime.python3.scripting.nodes/src/main/java/org/knime/python3/scripting/nodes2/PythonScriptingService.java @@ -68,6 +68,7 @@ import org.knime.core.node.CanceledExecutionException; import org.knime.core.node.ExecutionMonitor; import org.knime.core.node.NodeLogger; +import org.knime.core.node.port.PortObject; import org.knime.core.node.workflow.FlowObjectStack; import org.knime.core.node.workflow.NodeContext; import org.knime.core.util.ThreadUtils; @@ -75,6 +76,8 @@ import org.knime.core.webui.node.dialog.scripting.CodeGenerationRequest; import org.knime.core.webui.node.dialog.scripting.InputOutputModel; import org.knime.core.webui.node.dialog.scripting.ScriptingService; +import org.knime.externalprocessprovider.ExternalProcessProvider; +import org.knime.python3.scripting.nodes.PortsConfigurationUtils; import org.knime.python3.scripting.nodes2.PythonScriptingService.ExecutableOption.ExecutableOptionType; import org.knime.python3.scripting.nodes2.PythonScriptingSession.ExecutionInfo; import org.knime.python3.scripting.nodes2.PythonScriptingSession.ExecutionStatus; @@ -246,15 +249,30 @@ private void startNewInteractiveSession() throws IOException, InterruptedExcepti // Start the interactive Python session and setup the IO final var workflowControl = getWorkflowControl(); - final var pythonCommand = - ExecutableSelectionUtils.getPythonCommand(getExecutableOption(m_executableSelection)); + final var inputData = workflowControl.getInputData(); + + // Check if environment port is connected and extract Python command + ExternalProcessProvider pythonCommand = null; + PortObject[] dataPortObjects = inputData; // By default, all inputs are data ports + + if (m_ports.hasPythonEnvironmentPort() && inputData != null && inputData.length > 0) { + // TODO only log about environment installation if it actually needs to be installed + addConsoleOutputEvent( + new ConsoleText("Installing Python environment from environment port...\n", false)); + var envPort = PortsConfigurationUtils.extractPythonEnvironmentPort(inputData); + envPort.installPythonEnvironment(new ExecutionMonitor()); // Do not report the progress + addConsoleOutputEvent( + new ConsoleText("Successfully installed Python environment from environment port.\n", false)); + pythonCommand = PortsConfigurationUtils.extractPythonEnvironmentPort(inputData).getPythonCommand(); + } else { + pythonCommand = ExecutableSelectionUtils.getPythonCommand(getExecutableOption(m_executableSelection)); + } // TODO report the progress of converting the tables using the ExecutionMonitor? m_interactiveSession = new PythonScriptingSession(pythonCommand, PythonScriptingService.this::addConsoleOutputEvent, new DialogFileStoreHandlerSupplier()); - m_interactiveSession.setupIO(workflowControl.getInputData(), getSupportedFlowVariables(), - m_ports.getNumOutTables(), m_ports.getNumOutImages(), m_ports.getNumOutObjects(), m_hasView, - new ExecutionMonitor()); + m_interactiveSession.setupIO(dataPortObjects, getSupportedFlowVariables(), m_ports.getNumOutTables(), + m_ports.getNumOutImages(), m_ports.getNumOutObjects(), m_hasView, new ExecutionMonitor()); } private synchronized void executeScriptInternal(final String script, final boolean newSession, @@ -262,7 +280,18 @@ private synchronized void executeScriptInternal(final String script, final boole try { // Restart the session if necessary if (m_interactiveSession == null || newSession) { - startNewInteractiveSession(); + try { + startNewInteractiveSession(); + } catch (IOException | InterruptedException | CanceledExecutionException ex) { + // TODO cleanup + if (ex instanceof InterruptedException) { + Thread.currentThread().interrupt(); // Re-interrupt + } + var message = "Failed to start interactive Python session: " + ex.getMessage(); + LOGGER.error(message, ex); + sendExecutionFinishedEvent(new ExecutionInfo(ExecutionStatus.FATAL_ERROR, message)); + return; + } } // Run the script @@ -366,7 +395,7 @@ public String getLanguageServerConfig(final String executableSelection) { var executableOption = getExecutableOption(executableSelection); String executablePath = null; if (executableOption.type != ExecutableOptionType.MISSING_VAR) { - executablePath = ExecutableSelectionUtils.getPythonCommand(executableOption).getPythonExecutablePath() + executablePath = ExecutableSelectionUtils.getPythonCommand(executableOption).getExecutablePath() .toAbsolutePath().toString(); } var extraPaths = PythonScriptingSession.getExtraPythonPaths().stream() // diff --git a/org.knime.python3.scripting.nodes/src/main/java/org/knime/python3/scripting/nodes2/PythonScriptingSession.java b/org.knime.python3.scripting.nodes/src/main/java/org/knime/python3/scripting/nodes2/PythonScriptingSession.java index e520c4351..0929ca279 100644 --- a/org.knime.python3.scripting.nodes/src/main/java/org/knime/python3/scripting/nodes2/PythonScriptingSession.java +++ b/org.knime.python3.scripting.nodes/src/main/java/org/knime/python3/scripting/nodes2/PythonScriptingSession.java @@ -85,9 +85,9 @@ import org.knime.core.util.asynclose.AsynchronousCloseable; import org.knime.core.util.pathresolve.ResolverUtil; import org.knime.core.webui.node.dialog.scripting.ScriptingService.ConsoleText; +import org.knime.externalprocessprovider.ExternalProcessProvider; import org.knime.python3.Activator; import org.knime.python3.Python3SourceDirectory; -import org.knime.python3.PythonCommand; import org.knime.python3.PythonEntryPointUtils; import org.knime.python3.PythonFileStoreUtils; import org.knime.python3.PythonGateway; @@ -155,7 +155,7 @@ final class PythonScriptingSession implements AsynchronousCloseable private int m_numOutObjects; - PythonScriptingSession(final PythonCommand pythonCommand, final Consumer consoleTextConsumer, + PythonScriptingSession(final ExternalProcessProvider pythonCommand, final Consumer consoleTextConsumer, final FileStoreHandlerSupplier fileStoreHandlerSupplier) throws IOException, InterruptedException { m_consoleTextConsumer = consoleTextConsumer; m_fileStoreHandlerSupplier = fileStoreHandlerSupplier; @@ -418,10 +418,9 @@ Optional getOutputView() throws IOException { } } - private static PythonGateway createGateway(final PythonCommand pythonCommand) + private static PythonGateway createGateway(final ExternalProcessProvider pythonCommand) throws IOException, InterruptedException { - if (pythonCommand.getPythonExecutablePath() - .startsWith(CondaEnvironmentIdentifier.NOT_EXECUTED_PATH_PLACEHOLDER)) { + if (pythonCommand.getExecutablePath().startsWith(CondaEnvironmentIdentifier.NOT_EXECUTED_PATH_PLACEHOLDER)) { throw new IOException(CondaEnvironmentIdentifier.NOT_EXECUTED_PATH_PLACEHOLDER); } diff --git a/org.knime.python3.scripting.nodes/src/main/java/org/knime/python3/scripting/nodes2/script/PythonScriptNodeFactory.java b/org.knime.python3.scripting.nodes/src/main/java/org/knime/python3/scripting/nodes2/script/PythonScriptNodeFactory.java index 756f3bf9e..c719630ee 100644 --- a/org.knime.python3.scripting.nodes/src/main/java/org/knime/python3/scripting/nodes2/script/PythonScriptNodeFactory.java +++ b/org.knime.python3.scripting.nodes/src/main/java/org/knime/python3/scripting/nodes2/script/PythonScriptNodeFactory.java @@ -53,6 +53,7 @@ import static org.knime.python3.scripting.nodes2.PythonScriptPortsConfiguration.PORTGR_ID_OUT_IMAGE; import static org.knime.python3.scripting.nodes2.PythonScriptPortsConfiguration.PORTGR_ID_OUT_OBJECT; import static org.knime.python3.scripting.nodes2.PythonScriptPortsConfiguration.PORTGR_ID_OUT_TABLE; +import static org.knime.python3.scripting.nodes2.PythonScriptPortsConfiguration.PORTGR_ID_PYTHON_ENV; import java.util.Optional; @@ -66,6 +67,7 @@ import org.knime.core.webui.node.dialog.NodeDialog; import org.knime.core.webui.node.dialog.NodeDialogFactory; import org.knime.core.webui.node.dialog.NodeDialogManager; +import org.knime.pixi.port.PythonEnvironmentPortObject; import org.knime.python2.port.PickledObjectFileStorePortObject; import org.knime.python3.scripting.nodes2.PythonScriptNodeDialog; import org.knime.python3.scripting.nodes2.PythonScriptNodeModel; @@ -118,6 +120,10 @@ protected Optional createPortsConfigBuilder() { b.addExtendableInputPortGroup(PORTGR_ID_INP_OBJECT, PickledObjectFileStorePortObject.TYPE); b.addExtendableInputPortGroupWithDefault(PORTGR_ID_INP_TABLE, new PortType[0], new PortType[]{BufferedDataTable.TYPE}, BufferedDataTable.TYPE); + + // Add Python environment port + b.addOptionalInputPortGroup(PORTGR_ID_PYTHON_ENV, PythonEnvironmentPortObject.TYPE_OPTIONAL); + b.addExtendableOutputPortGroupWithDefault(PORTGR_ID_OUT_TABLE, new PortType[0], new PortType[]{BufferedDataTable.TYPE}, BufferedDataTable.TYPE); b.addExtendableOutputPortGroup(PORTGR_ID_OUT_IMAGE, ImagePortObject.TYPE); diff --git a/org.knime.python3.scripting.nodes/src/main/java/org/knime/python3/scripting/nodes2/view/PythonViewNodeFactory.java b/org.knime.python3.scripting.nodes/src/main/java/org/knime/python3/scripting/nodes2/view/PythonViewNodeFactory.java index 43eb35c8c..08239197d 100644 --- a/org.knime.python3.scripting.nodes/src/main/java/org/knime/python3/scripting/nodes2/view/PythonViewNodeFactory.java +++ b/org.knime.python3.scripting.nodes/src/main/java/org/knime/python3/scripting/nodes2/view/PythonViewNodeFactory.java @@ -51,6 +51,7 @@ import static org.knime.python3.scripting.nodes2.PythonScriptPortsConfiguration.PORTGR_ID_INP_OBJECT; import static org.knime.python3.scripting.nodes2.PythonScriptPortsConfiguration.PORTGR_ID_INP_TABLE; import static org.knime.python3.scripting.nodes2.PythonScriptPortsConfiguration.PORTGR_ID_OUT_IMAGE; +import static org.knime.python3.scripting.nodes2.PythonScriptPortsConfiguration.PORTGR_ID_PYTHON_ENV; import java.util.Optional; @@ -65,6 +66,7 @@ import org.knime.core.webui.node.dialog.NodeDialogManager; import org.knime.core.webui.node.view.NodeView; import org.knime.core.webui.node.view.NodeViewFactory; +import org.knime.pixi.port.PythonEnvironmentPortObject; import org.knime.python2.port.PickledObjectFileStorePortObject; import org.knime.python3.scripting.nodes2.PythonScriptNodeDialog; import org.knime.python3.scripting.nodes2.PythonScriptNodeModel; @@ -127,6 +129,10 @@ protected Optional createPortsConfigBuilder() { b.addExtendableInputPortGroup(PORTGR_ID_INP_OBJECT, PickledObjectFileStorePortObject.TYPE); b.addExtendableInputPortGroupWithDefault(PORTGR_ID_INP_TABLE, new PortType[0], new PortType[]{BufferedDataTable.TYPE}, BufferedDataTable.TYPE); + + // Add Python environment port + b.addOptionalInputPortGroup(PORTGR_ID_PYTHON_ENV, PythonEnvironmentPortObject.TYPE_OPTIONAL); + b.addOptionalOutputPortGroup(PORTGR_ID_OUT_IMAGE, ImagePortObject.TYPE); return Optional.of(b); } diff --git a/org.knime.python3.scripting.tests/META-INF/MANIFEST.MF b/org.knime.python3.scripting.tests/META-INF/MANIFEST.MF index 5a1b1e50a..336ac68dd 100644 --- a/org.knime.python3.scripting.tests/META-INF/MANIFEST.MF +++ b/org.knime.python3.scripting.tests/META-INF/MANIFEST.MF @@ -11,4 +11,5 @@ Automatic-Module-Name: org.knime.python3.scripting.tests Export-Package: org.knime.python3.scripting Require-Bundle: org.junit;bundle-version="[4.13.0,5.0.0)", org.apache.arrow.memory-core;bundle-version="[18.1.0,19.0.0)", - org.knime.python3.testing;bundle-version="[5.6.0,6.0.0)" + org.knime.python3.testing;bundle-version="[5.6.0,6.0.0)", + org.knime.externalprocessprovider;bundle-version="[5.11.0,6.0.0)" diff --git a/org.knime.python3.scripting.tests/src/test/java/org/knime/python3/scripting/Python3KernelBackendProxyTest.java b/org.knime.python3.scripting.tests/src/test/java/org/knime/python3/scripting/Python3KernelBackendProxyTest.java index aeef61a92..aa247c6d1 100644 --- a/org.knime.python3.scripting.tests/src/test/java/org/knime/python3/scripting/Python3KernelBackendProxyTest.java +++ b/org.knime.python3.scripting.tests/src/test/java/org/knime/python3/scripting/Python3KernelBackendProxyTest.java @@ -81,6 +81,7 @@ import org.knime.core.columnar.data.StringData.StringWriteData; import org.knime.core.table.schema.ColumnarSchema; import org.knime.core.util.FileUtil; +import org.knime.externalprocessprovider.ExternalProcessProvider; import org.knime.python2.kernel.PythonKernelBackendUtils; import org.knime.python3.DefaultPythonGateway; import org.knime.python3.Python3SourceDirectory; diff --git a/org.knime.python3.testing/META-INF/MANIFEST.MF b/org.knime.python3.testing/META-INF/MANIFEST.MF index 28ca50d58..4f5fa5456 100644 --- a/org.knime.python3.testing/META-INF/MANIFEST.MF +++ b/org.knime.python3.testing/META-INF/MANIFEST.MF @@ -8,6 +8,7 @@ Bundle-Vendor: KNIME AG, Zurich, Switzerland Bundle-RequiredExecutionEnvironment: JavaSE-17 Require-Bundle: org.apache.commons.lang3;bundle-version="[3.9.0,4.0.0)", org.knime.python3;bundle-version="[5.6.0,6.0.0)", - org.knime.core.columnar;bundle-version="[5.6.0,6.0.0)" + org.knime.core.columnar;bundle-version="[5.6.0,6.0.0)", + org.knime.externalprocessprovider;bundle-version="[5.11.0,6.0.0)" Automatic-Module-Name: org.knime.python3.testing Export-Package: org.knime.python3.testing diff --git a/org.knime.python3.testing/src/main/java/org/knime/python3/testing/Python3TestUtils.java b/org.knime.python3.testing/src/main/java/org/knime/python3/testing/Python3TestUtils.java index 7db4d2d13..8f0fb5346 100644 --- a/org.knime.python3.testing/src/main/java/org/knime/python3/testing/Python3TestUtils.java +++ b/org.knime.python3.testing/src/main/java/org/knime/python3/testing/Python3TestUtils.java @@ -51,7 +51,7 @@ import java.io.IOException; import org.apache.commons.lang3.SystemUtils; -import org.knime.python3.PythonCommand; +import org.knime.externalprocessprovider.ExternalProcessProvider; import org.knime.python3.SimplePythonCommand; /** @@ -74,7 +74,7 @@ private Python3TestUtils() { * @return The command created from environment variable. * @throws IOException If none of the environment variables is set. */ - public static PythonCommand getPythonCommand() throws IOException { + public static ExternalProcessProvider getPythonCommand() throws IOException { final String osSuffix; if (SystemUtils.IS_OS_LINUX) { osSuffix = "LINUX"; diff --git a/org.knime.python3/META-INF/MANIFEST.MF b/org.knime.python3/META-INF/MANIFEST.MF index c8075592b..743415d30 100644 --- a/org.knime.python3/META-INF/MANIFEST.MF +++ b/org.knime.python3/META-INF/MANIFEST.MF @@ -16,7 +16,8 @@ Require-Bundle: com.google.guava;bundle-version="[31.0.1,32.0.0)", org.knime.conda;bundle-version="[5.9.0,6.0.0)", org.knime.python3.types;bundle-version="[5.9.0,6.0.0)", org.eclipse.equinox.p2.core;bundle-version="[2.0.0,3.0.0)";visibility:=reexport, - org.eclipse.equinox.p2.engine;bundle-version="[2.0.0,3.0.0)";visibility:=reexport + org.eclipse.equinox.p2.engine;bundle-version="[2.0.0,3.0.0)";visibility:=reexport, + org.knime.externalprocessprovider;bundle-version="[5.11.0,6.0.0)" Export-Package: org.knime.python3, org.knime.python3.utils Automatic-Module-Name: org.knime.python3 diff --git a/org.knime.python3/src/main/java/org/knime/python3/AbstractPythonCommand.java b/org.knime.python3/src/main/java/org/knime/python3/AbstractPythonCommand.java index 64d2d2e5f..e03a9793c 100644 --- a/org.knime.python3/src/main/java/org/knime/python3/AbstractPythonCommand.java +++ b/org.knime.python3/src/main/java/org/knime/python3/AbstractPythonCommand.java @@ -54,12 +54,14 @@ import java.util.List; import java.util.Objects; +import org.knime.externalprocessprovider.ExternalProcessProvider; + /** * @author Marcel Wiedenmann, KNIME GmbH, Konstanz, Germany * @author Christian Dietz, KNIME GmbH, Konstanz, Germany * @author Benjamin Wilhelm, KNIME GmbH, Konstanz, Germany */ -abstract class AbstractPythonCommand implements PythonCommand { +abstract class AbstractPythonCommand implements ExternalProcessProvider { /** The Python command and possible arguments */ protected final List m_command; @@ -75,7 +77,7 @@ public ProcessBuilder createProcessBuilder() { } @Override - public Path getPythonExecutablePath() { + public Path getExecutablePath() { return Path.of(m_command.get(0)); } diff --git a/org.knime.python3/src/main/java/org/knime/python3/BundledPythonCommand.java b/org.knime.python3/src/main/java/org/knime/python3/BundledPythonCommand.java index 2000811f6..50962f85a 100644 --- a/org.knime.python3/src/main/java/org/knime/python3/BundledPythonCommand.java +++ b/org.knime.python3/src/main/java/org/knime/python3/BundledPythonCommand.java @@ -51,7 +51,7 @@ import org.knime.conda.CondaEnvironmentDirectory; /** - * Conda-specific implementation of {@link PythonCommand} that works with bundled Python environments. Allows to build + * Conda-specific implementation of {@link ExternalProcessProvider} that works with bundled Python environments. Allows to build * Python processes for a given Conda environment. Takes care of resolving PATH-related issues on Windows. * * @author Marcel Wiedenmann, KNIME GmbH, Konstanz, Germany @@ -60,7 +60,7 @@ public final class BundledPythonCommand extends AbstractCondaPythonCommand { /** - * Constructs a {@link PythonCommand} that describes a Python process that is run in the bundled Conda environment + * Constructs a {@link ExternalProcessProvider} that describes a Python process that is run in the bundled Conda environment * identified by the given Conda environment directory. The validity of the given argument is not tested. * * @param environmentDirectoryPath The path to the directory of the bundled Conda environment. diff --git a/org.knime.python3/src/main/java/org/knime/python3/CondaPythonCommand.java b/org.knime.python3/src/main/java/org/knime/python3/CondaPythonCommand.java index 37879c787..1381006d8 100644 --- a/org.knime.python3/src/main/java/org/knime/python3/CondaPythonCommand.java +++ b/org.knime.python3/src/main/java/org/knime/python3/CondaPythonCommand.java @@ -51,7 +51,7 @@ import org.knime.conda.CondaEnvironmentDirectory; /** - * Conda-specific implementation of {@link PythonCommand}. Allows to build Python processes for a given Conda + * Conda-specific implementation of {@link ExternalProcessProvider}. Allows to build Python processes for a given Conda * installation and environment. Takes care of resolving PATH-related issues on Windows. * * @author Marcel Wiedenmann, KNIME GmbH, Konstanz, Germany @@ -60,7 +60,7 @@ public final class CondaPythonCommand extends AbstractCondaPythonCommand { /** - * Constructs a {@link PythonCommand} that describes a Python process that is run in the Conda environment + * Constructs a {@link ExternalProcessProvider} that describes a Python process that is run in the Conda environment * identified by the given Conda installation directory and the given Conda environment directory.
* The validity of the given arguments is not tested. * diff --git a/org.knime.python3/src/main/java/org/knime/python3/PythonCommand.java b/org.knime.python3/src/main/java/org/knime/python3/PythonCommand.java index 6bb909f7d..32b5aecfc 100644 --- a/org.knime.python3/src/main/java/org/knime/python3/PythonCommand.java +++ b/org.knime.python3/src/main/java/org/knime/python3/PythonCommand.java @@ -50,6 +50,8 @@ import java.nio.file.Path; +import org.knime.externalprocessprovider.ExternalProcessProvider; + /** * Describes an external Python process. The process can be started via the {@link ProcessBuilder} returned by * {@link #createProcessBuilder()}. @@ -60,27 +62,40 @@ * @author Marcel Wiedenmann, KNIME GmbH, Konstanz, Germany * @author Christian Dietz, KNIME GmbH, Konstanz, Germany * @author Benjamin Wilhelm, KNIME GmbH, Konstanz, Germany + * @deprecated Use {@link ExternalProcessProvider} instead. This interface is kept for backward compatibility. */ -public interface PythonCommand { +@Deprecated(since = "5.11", forRemoval = true) +public interface PythonCommand extends ExternalProcessProvider { /** * @return A {@link ProcessBuilder} that can be used to parameterize and start the Python process represented by * this command instance. */ + @Deprecated(since = "5.11", forRemoval = true) + @Override ProcessBuilder createProcessBuilder(); /** * @return The path to the Python executable. Should only be used to gather information about the Python environment * without running the Python executable. Use {@link #createProcessBuilder()} to start Python processes. */ + @Deprecated(since = "5.11", forRemoval = true) Path getPythonExecutablePath(); + @Override + default Path getExecutablePath() { + return getPythonExecutablePath(); + } + + @Deprecated(since = "5.11", forRemoval = true) @Override int hashCode(); + @Deprecated(since = "5.11", forRemoval = true) @Override boolean equals(Object obj); + @Deprecated(since = "5.11", forRemoval = true) @Override String toString(); } diff --git a/org.knime.python3/src/main/java/org/knime/python3/PythonGatewayFactory.java b/org.knime.python3/src/main/java/org/knime/python3/PythonGatewayFactory.java index 79552f363..576cb765b 100644 --- a/org.knime.python3/src/main/java/org/knime/python3/PythonGatewayFactory.java +++ b/org.knime.python3/src/main/java/org/knime/python3/PythonGatewayFactory.java @@ -54,6 +54,7 @@ import java.util.List; import java.util.Objects; +import org.knime.externalprocessprovider.ExternalProcessProvider; import org.knime.python3.PythonPath.PythonPathBuilder; /** @@ -105,7 +106,7 @@ interface EntryPointCustomizer { */ final class PythonGatewayDescription { - private final PythonCommand m_command; + private final ExternalProcessProvider m_command; private final Path m_launcherPath; @@ -130,7 +131,7 @@ Path getLauncherPath() { return m_launcherPath; } - PythonCommand getCommand() { + ExternalProcessProvider getCommand() { return m_command; } @@ -188,7 +189,7 @@ public boolean equals(final Object obj) { * @param entryPointClass the type of entry point * @return a builder for a PythonGatewayDescription */ - public static Builder builder(final PythonCommand pythonCommand, + public static Builder builder(final ExternalProcessProvider pythonCommand, final Path launcherPath, final Class entryPointClass) { return new Builder<>(pythonCommand, launcherPath, entryPointClass); } @@ -203,7 +204,7 @@ public static final class Builder { private final Path m_launcherPath; - private final PythonCommand m_pythonCommand; + private final ExternalProcessProvider m_pythonCommand; private final Class m_entryPointClass; @@ -213,7 +214,7 @@ public static final class Builder { private final List> m_entryPointCustomizers = new ArrayList<>(); - private Builder(final PythonCommand pythonCommand, final Path launcherPath, + private Builder(final ExternalProcessProvider pythonCommand, final Path launcherPath, final Class entryPointClass) { m_launcherPath = launcherPath; m_pythonCommand = pythonCommand; diff --git a/org.knime.python3/src/main/java/org/knime/python3/QueuedPythonGatewayFactory.java b/org.knime.python3/src/main/java/org/knime/python3/QueuedPythonGatewayFactory.java index ad7ccf403..e0b2965c8 100644 --- a/org.knime.python3/src/main/java/org/knime/python3/QueuedPythonGatewayFactory.java +++ b/org.knime.python3/src/main/java/org/knime/python3/QueuedPythonGatewayFactory.java @@ -72,6 +72,7 @@ import java.util.stream.Collectors; import org.knime.core.node.NodeLogger; +import org.knime.externalprocessprovider.ExternalProcessProvider; import org.knime.python3.PythonGatewayCreationGate.PythonGatewayCreationGateListener; import com.google.common.util.concurrent.ThreadFactoryBuilder; @@ -153,7 +154,7 @@ public synchronized void reconfigureQueue(final int maxNumberOfIdlingGateways, * * @param command The Python command whose corresponding gateways to remove from the queue */ - public synchronized void clearQueuedGateways(final PythonCommand command) { + public synchronized void clearQueuedGateways(final ExternalProcessProvider command) { if (m_queue != null) { m_queue.clearQueuedGateways(command); } @@ -316,7 +317,7 @@ private BlockingQueue getGatewayQueue(final PythonGatewayDescript } @Override - public synchronized void clearQueuedGateways(final PythonCommand command) { + public synchronized void clearQueuedGateways(final ExternalProcessProvider command) { final List gatewaysToEvict = new ArrayList<>(); for (final var entry : m_gateways.entrySet()) { if (entry.getKey().getCommand().equals(command)) { @@ -467,7 +468,7 @@ public PythonGatewayDummyQueue(final int maxNumberOfIdlingGateways, final int ex } @Override - public void clearQueuedGateways(final PythonCommand command) { + public void clearQueuedGateways(final ExternalProcessProvider command) { // Nothing to do. } @@ -496,11 +497,11 @@ public AbstractPythonGatewayQueue(final int maxNumberOfIdlingGateways, final int getNextGateway(PythonGatewayDescription description) throws IOException, InterruptedException; /** - * Clears all queued gateways that were created with the specified {@link PythonCommand}. + * Clears all queued gateways that were created with the specified {@link ExternalProcessProvider}. * - * @param command The {@link PythonCommand} + * @param command The {@link ExternalProcessProvider} */ - public abstract void clearQueuedGateways(PythonCommand command); + public abstract void clearQueuedGateways(ExternalProcessProvider command); @Override public abstract void close(); diff --git a/org.knime.python3/src/main/java/org/knime/python3/SimplePythonCommand.java b/org.knime.python3/src/main/java/org/knime/python3/SimplePythonCommand.java index 0475d7fa9..eb8b3b640 100644 --- a/org.knime.python3/src/main/java/org/knime/python3/SimplePythonCommand.java +++ b/org.knime.python3/src/main/java/org/knime/python3/SimplePythonCommand.java @@ -51,7 +51,7 @@ import java.util.List; /** - * A simple implementation of {@link PythonCommand}. Runs a command that is given by a list of strings. + * A simple implementation of {@link ExternalProcessProvider}. Runs a command that is given by a list of strings. * * @author Marcel Wiedenmann, KNIME GmbH, Konstanz, Germany * @author Christian Dietz, KNIME GmbH, Konstanz, Germany