Enable user functions in configurations

Author: SteveL-MSFT

dsc/tests/dsc_user_functions.tests.ps1

@@ -0,0 +1,133 @@
+# Copyright (c) Microsoft Corporation.
+# Licensed under the MIT License.
+
+Describe 'user function tests' {
+    It 'user function working with expression: <expression>' -TestCases @(
+        @{ expression = "[MyFunction.ComboFunction('test', 42, true)]"; expected = 'test-42-True' }
+        @{ expression = "[MyOtherNamespace.ArrayFunction(createArray('a','b','c','d'))]"; expected = @('["b","c","d"]-a') }
+    ) {
+        param($expression, $expected)
+
+        $configYaml = @"
+`$schema: https://aka.ms/dsc/schemas/v3/bundled/config/document.json
+functions:
+- namespace: MyFunction
+  members:
+    ComboFunction:
+      parameters:
+      - name: StringParam
+        type: string
+      - name: NumberParam
+        type: int
+      - name: BoolParam
+        type: bool
+      output:
+        type: string
+        value: "[format('{0}-{1}-{2}', parameters('StringParam'), parameters('NumberParam'), parameters('BoolParam'))]"
+- namespace: MyOtherNamespace
+  members:
+    ArrayFunction:
+      parameters:
+      - name: ArrayParam
+        type: array
+      output:
+        type: array
+        value: "[array(format('{0}-{1}', string(skip(parameters('ArrayParam'),1)), first(parameters('ArrayParam'))))]"
+resources:
+- name: test
+  type: Microsoft.DSC.Debug/Echo
+  properties:
+    output: "$expression"
+"@
+
+        $out = dsc -l trace config get -i $configYaml 2>$testdrive/error.log | ConvertFrom-Json
+        $LASTEXITCODE | Should -Be 0 -Because (Get-Content $testdrive/error.log | Out-String)
+        $out.results[0].result.actualState.output | Should -Be $expected -Because ($out | ConvertTo-Json -Depth 10 | Out-String)
+    }
+
+    It 'user function returning object works' {
+        $configYaml = @"
+`$schema: https://aka.ms/dsc/schemas/v3/bundled/config/document.json
+functions:
+- namespace: MyObjectFunction
+  members:
+    ObjectFunction:
+      parameters:
+      - name: ObjectParam
+        type: object
+      output:
+        type: object
+        value: "[createObject('myKey', concat('#', string(parameters('ObjectParam'))))]"
+resources:
+- name: test
+  type: Microsoft.DSC.Debug/Echo
+  properties:
+    output: "[MyObjectFunction.ObjectFunction(createObject('key','value'))]"
+"@
+
+        $out = dsc -l trace config get -i $configYaml 2>$testdrive/error.log | ConvertFrom-Json
+        $LASTEXITCODE | Should -Be 0 -Because (Get-Content $testdrive/error.log | Out-String)
+        $out.results[0].result.actualState.output.myKey | Should -Be '#{"key":"value"}' -Because ($out | ConvertTo-Json -Depth 10 | Out-String)
+    }
+
+    It 'user functions cannot call function with expression: <expression>' -TestCases @(
+        @{ expression = "[reference('foo/bar')]"; errorText = "The 'reference()' function is not available in user-defined functions" }
+        @{ expression = "[utcNow()]"; errorText = "The 'utcNow()' function can only be used as a parameter default" }
+        @{ expression = "[variables('myVar')]"; errorText = "The 'variables()' function is not available in user-defined functions" }
+        @{ expression = "[MyFunction.OtherFunction()]"; errorText = "Unknown user function: MyFunction.OtherFunctio" }
+        @{ expression = "[MyFunction.BadFunction()]"; errorText = "Unknown user function: MyFunction.BadFunction" }
+    ) {
+        param($expression, $errorText)
+
+        $configYaml = @"
+`$schema: https://aka.ms/dsc/schemas/v3/bundled/config/document.json
+variables:
+  myVar: someValue
+functions:
+- namespace: MyFunction
+  members:
+    BadFunction:
+      output:
+        type: string
+        value: "$expression"
+    OtherFunction:
+      output:
+        type: string
+        value: "test"
+resources:
+- name: test
+  type: Microsoft.DSC.Debug/Echo
+  properties:
+    output: "[MyFunction.BadFunction()]"
+"@
+
+        dsc -l trace config get -i $configYaml 2>$testdrive/error.log | Out-Null
+        $LASTEXITCODE | Should -Be 2 -Because (Get-Content $testdrive/error.log | Out-String)
+        (Get-Content $testdrive/error.log -Raw) | Should -BeLike "*$errorText*" -Because (Get-Content $testdrive/error.log | Out-String)
+    }
+
+    It 'user function with invalid parameter fails' {
+        $configYaml = @"
+`$schema: https://aka.ms/dsc/schemas/v3/bundled/config/document.json
+functions:
+- namespace: MyFunction
+  members:
+    BadFunction:
+      parameters:
+      - name: Param1
+        type: string
+      output:
+        type: string
+        value: "[parameters('BadParam')]"
+resources:
+- name: test
+  type: Microsoft.DSC.Debug/Echo
+  properties:
+    output: "[MyFunction.BadFunction('test')]"
+"@
+
+        dsc -l trace config get -i $configYaml 2>$testdrive/error.log | Out-Null
+        $LASTEXITCODE | Should -Be 2 -Because (Get-Content $testdrive/error.log | Out-String)
+        (Get-Content $testdrive/error.log -Raw) | Should -BeLike "*Parameter 'BadParam' not found in context*" -Because (Get-Content $testdrive/error.log | Out-String)
+    }
+}

dsc_lib/locales/en-us.toml

@@ -76,6 +76,8 @@ unrollingCopy = "Unrolling copy for resource '%{name}' with count %{count}"
 copyModeNotSupported = "Copy mode is not supported"
 copyBatchSizeNotSupported = "Copy batch size is not supported"
 copyNameResultNotString = "Copy name result is not a string"
+userFunctionAlreadyDefined = "User function '%{name}' in namespace '%{namespace}' is already defined"
+addingUserFunction = "Adding user function '%{name}'"
 
 [discovery.commandDiscovery]
 couldNotReadSetting = "Could not read 'resourcePath' setting"
@@ -426,6 +428,7 @@ description = "Retrieves the output of a previously executed resource"
 invoked = "reference function"
 keyNotFound = "Invalid resourceId or resource has not executed yet: %{key}"
 cannotUseInCopyMode = "The 'reference()' function cannot be used when processing a 'Copy' loop"
+unavailableInUserFunction = "The 'reference()' function is not available in user-defined functions"
 
 [functions.resourceId]
 description = "Constructs a resource ID from the given type and name"
@@ -479,12 +482,13 @@ invoked = "uniqueString function"
 [functions.utcNow]
 description = "Returns the current UTC time"
 invoked = "utcNow function"
-onlyUsedAsParameterDefault = "utcNow function can only be used as a parameter default"
+onlyUsedAsParameterDefault = "The 'utcNow()' function can only be used as a parameter default"
 
 [functions.variables]
 description = "Retrieves the value of a variable"
 invoked = "variables function"
 keyNotFound = "Variable '%{key}' does not exist or has not been initialized yet"
+unavailableInUserFunction = "The 'variables()' function is not available in user-defined functions"
 
 [parser.expression]
 functionNodeNotFound = "Function node not found"

dsc_lib/src/configure/config_doc.rs

@@ -105,6 +105,30 @@ pub struct Metadata {
     pub other: Map<String, Value>,
 }
 
+#[derive(Debug, Clone, PartialEq, Deserialize, Serialize, JsonSchema)]
+pub struct UserFunction {
+    pub namespace: String,
+    pub members: HashMap<String, UserFunctionDefinition>,
+}
+
+#[derive(Debug, Clone, PartialEq, Deserialize, Serialize, JsonSchema)]
+pub struct UserFunctionDefinition {
+    pub parameters: Option<Vec<UserFunctionParameter>>,
+    pub output: UserFunctionOutput,
+}
+
+#[derive(Debug, Clone, PartialEq, Deserialize, Serialize, JsonSchema)]
+pub struct UserFunctionParameter {
+    pub name: String,
+    pub r#type: DataType,
+}
+
+#[derive(Debug, Clone, PartialEq, Deserialize, Serialize, JsonSchema)]
+pub struct UserFunctionOutput {
+    pub r#type: DataType,
+    pub value: String,
+}
+
 #[derive(Debug, Clone, PartialEq, Deserialize, Serialize, JsonSchema)]
 #[serde(deny_unknown_fields)]
 pub struct Configuration {
@@ -114,6 +138,8 @@ pub struct Configuration {
     #[serde(rename = "contentVersion")]
     pub content_version: Option<String>,
     #[serde(skip_serializing_if = "Option::is_none")]
+    pub functions: Option<Vec<UserFunction>>,
+    #[serde(skip_serializing_if = "Option::is_none")]
     pub parameters: Option<HashMap<String, Parameter>>,
     #[serde(skip_serializing_if = "Option::is_none")]
     pub variables: Option<Map<String, Value>>,
@@ -296,10 +322,11 @@ impl Configuration {
         Self {
             schema: Self::default_schema_id_uri(),
             content_version: Some("1.0.0".to_string()),
+            metadata: None,
             parameters: None,
-            variables: None,
             resources: Vec::new(),
-            metadata: None,
+            functions: None,
+            variables: None,
         }
     }
 }

dsc_lib/src/configure/context.rs

@@ -2,7 +2,7 @@
 // Licensed under the MIT License.
 
 use chrono::{DateTime, Local};
-use crate::{configure::config_doc::ExecutionKind, extensions::dscextension::DscExtension};
+use crate::{configure::config_doc::{ExecutionKind, UserFunctionDefinition}, extensions::dscextension::DscExtension};
 use security_context_lib::{get_security_context, SecurityContext};
 use serde_json::{Map, Value};
 use std::{collections::HashMap, path::PathBuf};
@@ -34,6 +34,7 @@ pub struct Context {
     pub security_context: SecurityContextKind,
     pub start_datetime: DateTime<Local>,
     pub system_root: PathBuf,
+    pub user_functions: HashMap<String, UserFunctionDefinition>,
     pub variables: Map<String, Value>,
 }
 
@@ -58,6 +59,7 @@ impl Context {
             },
             start_datetime: chrono::Local::now(),
             system_root: get_default_os_system_root(),
+            user_functions: HashMap::new(),
             variables: Map::new(),
         }
     }

dsc_lib/src/configure/mod.rs

@@ -16,7 +16,7 @@ use crate::DscResource;
 use crate::discovery::Discovery;
 use crate::parser::Statement;
 use crate::progress::{Failure, ProgressBar, ProgressFormat};
-use self::context::Context;
+use self::context::{Context, ProcessMode};
 use self::config_doc::{Configuration, DataType, MicrosoftDscMetadata, Operation, SecurityContextKind};
 use self::depends_on::get_resource_invocation_order;
 use self::config_result::{ConfigurationExportResult, ConfigurationGetResult, ConfigurationSetResult, ConfigurationTestResult};
@@ -729,6 +729,7 @@ impl Configurator {
         self.context.extensions = self.discovery.extensions.values().cloned().collect();
         self.set_parameters(parameters_input, &config)?;
         self.set_variables(&config)?;
+        self.set_user_functions(&config)?;
         Ok(())
     }
 
@@ -749,9 +750,9 @@ impl Configurator {
                 // default values can be expressions
                 let value = if default_value.is_string() {
                     if let Some(value) = default_value.as_str() {
-                        self.context.processing_parameter_defaults = true;
+                        self.context.process_mode = ProcessMode::ParametersDefault;
                         let result = self.statement_parser.parse_and_execute(value, &self.context)?;
-                        self.context.processing_parameter_defaults = false;
+                        self.context.process_mode = ProcessMode::Normal;
                         result
                     } else {
                         return Err(DscError::Parser(t!("configure.mod.defaultStringNotDefined").to_string()));
@@ -824,6 +825,23 @@ impl Configurator {
         Ok(())
     }
 
+    fn set_user_functions(&mut self, config: &Configuration) -> Result<(), DscError> {
+        let Some(functions) = &config.functions else {
+            return Ok(());
+        };
+
+        for user_function in functions {
+            for (function_name, function_definition) in &user_function.members {
+                if self.context.user_functions.contains_key(&format!("{}.{}", user_function.namespace, function_name)) {
+                    return Err(DscError::Validation(t!("configure.mod.userFunctionAlreadyDefined", name = function_name, namespace = user_function.namespace).to_string()));
+                }
+                debug!("{}", t!("configure.mod.addingUserFunction", name = format!("{}.{}", user_function.namespace, function_name)));
+                self.context.user_functions.insert(format!("{}.{}", user_function.namespace.to_lowercase(), function_name.to_lowercase()), function_definition.clone());
+            }
+        }
+        Ok(())
+    }
+
     fn get_result_metadata(&self, operation: Operation) -> Metadata {
         let end_datetime = chrono::Local::now();
         let version = self

dsc_lib/src/dscresources/dscresource.rs

@@ -1,7 +1,7 @@
 // Copyright (c) Microsoft Corporation.
 // Licensed under the MIT License.
 
-use crate::{configure::{config_doc::{Configuration, ExecutionKind, Resource}, Configurator}, dscresources::resource_manifest::Kind};
+use crate::{configure::{config_doc::{Configuration, ExecutionKind, Resource}, Configurator, context::ProcessMode}, dscresources::resource_manifest::Kind};
 use crate::dscresources::invoke_result::{ResourceGetResponse, ResourceSetResponse};
 use dscerror::DscError;
 use jsonschema::Validator;
@@ -126,7 +126,7 @@ impl DscResource {
         let config_json = serde_json::to_string(&configuration)?;
         let mut configurator = Configurator::new(&config_json, crate::progress::ProgressFormat::None)?;
         // don't process expressions again as they would have already been processed before being passed to the adapter
-        configurator.context.process_expressions = false;
+        configurator.context.process_mode = ProcessMode::NoExpressionEvaluation;
         Ok(configurator)
     }
 }

dsc_lib/src/functions/mod.rs

@@ -5,6 +5,7 @@ use std::collections::HashMap;
 
 use crate::DscError;
 use crate::configure::context::Context;
+use crate::functions::user_function::invoke_user_function;
 use rust_i18n::t;
 use schemars::JsonSchema;
 use serde::Serialize;
@@ -60,6 +61,7 @@ pub mod system_root;
 pub mod r#true;
 pub mod union;
 pub mod unique_string;
+pub mod user_function;
 pub mod utc_now;
 pub mod variables;
 
@@ -197,6 +199,10 @@ impl FunctionDispatcher {
     /// This function will return an error if the function fails to execute.
     pub fn invoke(&self, name: &str, args: &[Value], context: &Context) -> Result<Value, DscError> {
         let Some(function) = self.functions.get(name) else {
+            // if function name contains a period, it might be a user function
+            if name.contains('.') {
+                return invoke_user_function(name, args, context);
+            }
             return Err(DscError::Parser(t!("functions.unknownFunction", name = name).to_string()));
         };
 
@@ -224,32 +230,32 @@ impl FunctionDispatcher {
                 break;
             }
 
-            Self::check_arg_against_expected_types(value, &metadata.accepted_arg_ordered_types[index])?;
+            Self::check_arg_against_expected_types(name, value, &metadata.accepted_arg_ordered_types[index])?;
         }
 
         // if we have remaining args, they must match one of the remaining_arg_types
         if let Some(remaining_arg_types) = metadata.remaining_arg_accepted_types {
             for value in args.iter().skip(metadata.accepted_arg_ordered_types.len()) {
-                Self::check_arg_against_expected_types(value, &remaining_arg_types)?;
+                Self::check_arg_against_expected_types(name, value, &remaining_arg_types)?;
             }
         }
 
         function.invoke(args, context)
     }
 
-    fn check_arg_against_expected_types(arg: &Value, expected_types: &[FunctionArgKind]) -> Result<(), DscError> {
+    fn check_arg_against_expected_types(name: &str, arg: &Value, expected_types: &[FunctionArgKind]) -> Result<(), DscError> {
         if arg.is_array() && !expected_types.contains(&FunctionArgKind::Array) {
-            return Err(DscError::Parser(t!("functions.noArrayArgs", accepted_args_string = expected_types.iter().map(std::string::ToString::to_string).collect::<Vec<_>>().join(", ")).to_string()));
+            return Err(DscError::Parser(t!("functions.noArrayArgs", name = name, accepted_args_string = expected_types.iter().map(std::string::ToString::to_string).collect::<Vec<_>>().join(", ")).to_string()));
         } else if arg.is_boolean() && !expected_types.contains(&FunctionArgKind::Boolean) {
-            return Err(DscError::Parser(t!("functions.noBooleanArgs", accepted_args_string = expected_types.iter().map(std::string::ToString::to_string).collect::<Vec<_>>().join(", ")).to_string()));
+            return Err(DscError::Parser(t!("functions.noBooleanArgs", name = name, accepted_args_string = expected_types.iter().map(std::string::ToString::to_string).collect::<Vec<_>>().join(", ")).to_string()));
         } else if arg.is_null() && !expected_types.contains(&FunctionArgKind::Null) {
-            return Err(DscError::Parser(t!("functions.noNullArgs", accepted_args_string = expected_types.iter().map(std::string::ToString::to_string).collect::<Vec<_>>().join(", ")).to_string()));
+            return Err(DscError::Parser(t!("functions.noNullArgs", name = name, accepted_args_string = expected_types.iter().map(std::string::ToString::to_string).collect::<Vec<_>>().join(", ")).to_string()));
         } else if arg.is_number() && !expected_types.contains(&FunctionArgKind::Number) {
-            return Err(DscError::Parser(t!("functions.noNumberArgs", accepted_args_string = expected_types.iter().map(std::string::ToString::to_string).collect::<Vec<_>>().join(", ")).to_string()));
+            return Err(DscError::Parser(t!("functions.noNumberArgs", name = name, accepted_args_string = expected_types.iter().map(std::string::ToString::to_string).collect::<Vec<_>>().join(", ")).to_string()));
         } else if arg.is_object() && !expected_types.contains(&FunctionArgKind::Object) {
-            return Err(DscError::Parser(t!("functions.noObjectArgs", accepted_args_string = expected_types.iter().map(std::string::ToString::to_string).collect::<Vec<_>>().join(", ")).to_string()));
+            return Err(DscError::Parser(t!("functions.noObjectArgs", name = name, accepted_args_string = expected_types.iter().map(std::string::ToString::to_string).collect::<Vec<_>>().join(", ")).to_string()));
         } else if arg.is_string() && !expected_types.contains(&FunctionArgKind::String) {
-            return Err(DscError::Parser(t!("functions.noStringArgs", accepted_args_string = expected_types.iter().map(std::string::ToString::to_string).collect::<Vec<_>>().join(", ")).to_string()));
+            return Err(DscError::Parser(t!("functions.noStringArgs", name = name, accepted_args_string = expected_types.iter().map(std::string::ToString::to_string).collect::<Vec<_>>().join(", ")).to_string()));
         }
         Ok(())
     }

dsc_lib/src/functions/reference.rs

@@ -37,6 +37,9 @@ impl Function for Reference {
         if context.process_mode == ProcessMode::Copy {
             return Err(DscError::Parser(t!("functions.reference.cannotUseInCopyMode").to_string()));
         }
+        if context.process_mode == ProcessMode::UserFunction {
+            return Err(DscError::Parser(t!("functions.reference.unavailableInUserFunction").to_string()));
+        }
 
         if let Some(key) = args[0].as_str() {
             if context.references.contains_key(key) {