Merge branch 'main' of https://github.com/Gijsreyn/operation-methods into add-array-functions

Author: Gijsreyn

.cargo/config.toml

@@ -19,8 +19,8 @@ rustflags = ["-Ccontrol-flow-guard", "-Ctarget-feature=+crt-static", "-Clink-arg
 [source.crates-io]
 replace-with = "POWERSHELL"
 
-[target.aarch64-unknown-linux-gnu]
-linker = "aarch64-linux-gnu-gcc"
+#[target.aarch64-unknown-linux-gnu]
+#linker = "aarch64-linux-gnu-gcc"
 
-[target.aarch64-unknown-linux-musl]
-linker = "aarch64-linux-musl-gcc"
+#[target.aarch64-unknown-linux-musl]
+#linker = "aarch64-linux-musl-gcc"

.pipelines/DSC-Official.yml

@@ -7,11 +7,25 @@ pr:
     - onebranch
     - release/v*
 
+parameters:
+  - name: OfficialBuild
+    type: boolean
+    default: false
+
 variables:
-  BuildConfiguration: 'release'
-  PackageRoot: '$(System.ArtifactsDirectory)/Packages'
-  WindowsContainerImage: 'onebranch.azurecr.io/windows/ltsc2022/vse2022:latest'
-  Codeql.TSAEnabled: true
+  - name: BuildConfiguration
+    value: 'release'
+  - name: PackageRoot
+    value: '$(System.ArtifactsDirectory)/Packages'
+  - name: WindowsContainerImage
+    value: 'onebranch.azurecr.io/windows/ltsc2022/vse2022:latest'
+  - name: Codeql.TSAEnabled
+    value: true
+  - name: templateFile
+    value: ${{ iif ( parameters.OfficialBuild, 'v2/OneBranch.Official.CrossPlat.yml@onebranchTemplates', 'v2/OneBranch.NonOfficial.CrossPlat.yml@onebranchTemplates' ) }}
+  - group: DSC-Rust.SDK
+  - name: officialBuild
+    value: ${{ parameters.OfficialBuild }}
 
 resources:
   repositories:
@@ -21,7 +35,7 @@ resources:
     ref: refs/heads/main
 
 extends:
-  template: v2/OneBranch.Official.CrossPlat.yml@onebranchTemplates
+  template: ${{ variables.templateFile }}
   parameters:
     featureFlags:
       WindowsHostVersion:
@@ -78,7 +92,7 @@ extends:
         - task: AzureCLI@2
           displayName: Get Az Token
           inputs:
-            azureSubscription: az-blob-cicd-infra
+            azureSubscription: PowerShell-CICD-Feed-Access
             scriptType: pscore
             scriptLocation: inlineScript
             inlineScript: |
@@ -105,14 +119,16 @@ extends:
             buildName: x86_64-pc-windows-msvc
             signSrcPath: '$(signSrcPath)'
             PackageRoot: '$(PackageRoot)'
-            token: '$(AzToken)'
+            aztoken: '$(AzToken)'
+            rustSDK: '$(Rust.SDK)'
 
       - job: BuildWin_arm64
         dependsOn: SetPackageVersion
         variables:
           ob_sdl_tsa_configFile: '$(Build.SourcesDirectory)\DSC\.config\tsaoptions.json'
           ob_outputDirectory: '$(Build.ArtifactStagingDirectory)'
           signSrcPath: '$(Build.SourcesDirectory)\out'
+          AzToken: $[ dependencies.SetPackageVersion.outputs['AzToken'] ]
           ob_sdl_sbom_enabled: true
           ob_signing_setup_enabled: true
           ob_sdl_codeql_compiled_enabled: true
@@ -124,7 +140,8 @@ extends:
             buildName: aarch64-pc-windows-msvc
             signSrcPath: '$(signSrcPath)'
             PackageRoot: '$(PackageRoot)'
-            token: '$(AzToken)'
+            aztoken: '$(AzToken)'
+            rustSDK: '$(Rust.SDK)'
 
       - job: CreateMsixBundle
         dependsOn:
@@ -162,6 +179,7 @@ extends:
           condition: succeeded()
 
       - job: SignMsixBundle
+        condition: and(succeeded(), eq(variables.officialBuild, true))
         dependsOn: CreateMsixBundle
         variables:
           ob_outputDirectory: '$(Build.ArtifactStagingDirectory)'
@@ -208,15 +226,15 @@ extends:
         - task: RustInstaller@1
           inputs:
             rustVersion: ms-stable
-            toolchainFeed: https://pkgs.dev.azure.com/mscodehub/Rust/_packaging/Rust/nuget/v3/index.json
+            toolchainFeed: $(Rust.SDK)
             additionalTargets: x86_64-unknown-linux-musl
           displayName: Install Rust
           env:
             ob_restore_phase: true
         - pwsh: |
             apt update
             apt -y install musl-tools
-            $header = "Bearer $accessToken"
+            $header = "Bearer $(AzToken)"
             $env:CARGO_REGISTRIES_POWERSHELL_TOKEN = $header
             $env:CARGO_REGISTRIES_POWERSHELL_CREDENTIAL_PROVIDER = 'cargo:token'
             ./build.ps1 -Release -Architecture x86_64-unknown-linux-musl
@@ -243,15 +261,15 @@ extends:
         - task: RustInstaller@1
           inputs:
             rustVersion: ms-stable
-            toolchainFeed: https://pkgs.dev.azure.com/mscodehub/Rust/_packaging/Rust/nuget/v3/index.json
+            toolchainFeed: $(Rust.SDK)
             additionalTargets: aarch64-unknown-linux-musl
           displayName: Install Rust
           env:
             ob_restore_phase: true
         - task: AzureCLI@2
           displayName: Azure CLI
           inputs:
-            azureSubscription: az-blob-cicd-infra
+            azureSubscription: PowerShell-CICD-Feed-Access
             scriptType: pscore
             scriptLocation: inlineScript
             inlineScript: |
@@ -264,14 +282,15 @@ extends:
             apt -y install clang
             apt -y install llvm
             apt -y install musl-tools
-            apt -y install gcc-multilib
+            apt -y install musl-gcc
+            #apt -y install gcc-multilib
             apt -y install libssl-dev
             apt -y install pkg-config
             msrustup default stable-aarch64-unknown-linux-musl
             if ((openssl version -d) -match 'OPENSSLDIR: "(?<dir>.*?)"') {
               $env:OPENSSL_LIB_DIR = $matches['dir']
             }
-            $header = "Bearer $accessToken"
+            $header = "Bearer $(AzToken)"
             $env:CARGO_REGISTRIES_POWERSHELL_TOKEN = $header
             $env:CARGO_REGISTRIES_POWERSHELL_CREDENTIAL_PROVIDER = 'cargo:token'
             ./build.ps1 -Release -Architecture aarch64-unknown-linux-musl
@@ -302,23 +321,24 @@ extends:
         - task: RustInstaller@1
           inputs:
             rustVersion: ms-stable
-            toolchainFeed: https://pkgs.dev.azure.com/mscodehub/Rust/_packaging/Rust/nuget/v3/index.json
+            toolchainFeed: $(Rust.SDK)
             additionalTargets: $(buildName)
           displayName: Install Rust
           env:
             ob_restore_phase: true
         - task: AzureCLI@2
           displayName: Azure CLI
           inputs:
-            azureSubscription: az-blob-cicd-infra
+            azureSubscription: PowerShell-CICD-Feed-Access
             scriptType: pscore
             scriptLocation: inlineScript
             inlineScript: |
               az account show
         - pwsh: |
-            $header = "Bearer $accessToken"
+            $header = "Bearer $(AzToken)"
             $env:CARGO_REGISTRIES_POWERSHELL_TOKEN = $header
             $env:CARGO_REGISTRIES_POWERSHELL_CREDENTIAL_PROVIDER = 'cargo:token'
+            Write-Verbose -Verbose "Building for $(buildName)"
             ./build.ps1 -Release -Architecture $(buildName)
             ./build.ps1 -PackageType tgz -Architecture $(buildName) -Release
             Copy-Item ./bin/*.tar.gz "$(ob_outputDirectory)"

.pipelines/DSC-Windows.yml

@@ -8,7 +8,9 @@ parameters:
   - name: BuildConfiguration
     type: string
     default: Release
-  - name: token
+  - name: aztoken
+    type: string
+  - name: RustSDK
     type: string
 
 steps:
@@ -31,13 +33,13 @@ steps:
 - task: RustInstaller@1
   inputs:
     rustVersion: ms-stable
-    toolchainFeed: https://pkgs.dev.azure.com/mscodehub/Rust/_packaging/Rust/nuget/v3/index.json
+    toolchainFeed: ${{ parameters.RustSDK }}
     additionalTargets: ${{ parameters.buildName }}
   displayName: Install Rust
   env:
     ob_restore_phase: true
 - pwsh: |
-    $header = "Bearer ${ parameters.token }"
+    $header = "Bearer ${{ parameters.aztoken }}"
     $env:CARGO_REGISTRIES_POWERSHELL_TOKEN = $header
     $env:CARGO_REGISTRIES_POWERSHELL_CREDENTIAL_PROVIDER = 'cargo:token'
     Set-Location "$(Build.SourcesDirectory)/DSC"
@@ -47,6 +49,7 @@ steps:
     }
     $env:PATH += ";$LLVMBIN"
     write-verbose -verbose (gcm clang.exe | out-string)
+    Write-Verbose -Verbose "Building for ${{ parameters.buildName }}"
     ./build.ps1 -Release -Architecture ${{ parameters.buildName }} -SkipLinkCheck
   displayName: 'Build ${{ parameters.buildName }}'
   env:

build.ps1

@@ -25,6 +25,7 @@ param(
 
 $env:RUSTC_LOG=$null
 $env:RUSTFLAGS='-Dwarnings'
+$usingADO = ($null -ne $env:TF_BUILD)
 
 trap {
     Write-Error "An error occurred: $($_ | Out-String)"
@@ -155,6 +156,7 @@ function Find-LinkExe {
 
 $channel = 'stable'
 if ($null -ne (Get-Command msrustup -CommandType Application -ErrorAction Ignore)) {
+    Write-Verbose -Verbose "Using msrustup"
     $rustup = 'msrustup'
     $channel = 'ms-stable'
     if ($architecture -eq 'current') {
@@ -170,7 +172,7 @@ if ($null -ne $packageType) {
     $SkipBuild = $true
 } else {
     ## Test if Rust is installed
-    if (!(Get-Command 'cargo' -ErrorAction Ignore)) {
+    if (!$usingADO -and !(Get-Command 'cargo' -ErrorAction Ignore)) {
         Write-Verbose -Verbose "Rust not found, installing..."
         if (!$IsWindows) {
             curl https://sh.rustup.rs -sSf | sh -s -- -y
@@ -183,15 +185,22 @@ if ($null -ne $packageType) {
             $env:PATH += ";$env:USERPROFILE\.cargo\bin"
             Remove-Item temp:/rustup-init.exe -ErrorAction Ignore
         }
+        if ($LASTEXITCODE -ne 0) {
+            throw "Failed to install Rust"
+        }
     }
-    else  {
+    elseif (!$usingADO) {
         Write-Verbose -Verbose "Rust found, updating..."
         & $rustup update
     }
 
-    $BuildToolsPath = "${env:ProgramFiles(x86)}\Microsoft Visual Studio\2022\BuildTools\VC\Tools\MSVC"
+    if ($IsWindows) {
+        $BuildToolsPath = "${env:ProgramFiles(x86)}\Microsoft Visual Studio\2022\BuildTools\VC\Tools\MSVC"
+    }
 
-    & $rustup default stable
+    if (!$usingADO) {
+        & $rustup default stable
+    }
 
     ## Test if Node is installed
     ## Skipping upgrade as users may have a specific version they want to use
@@ -212,6 +221,18 @@ if ($null -ne $packageType) {
                 Write-Warning "winget not found, please install Node.js manually"
             }
         }
+        if ($LASTEXITCODE -ne 0) {
+            throw "Failed to install Node.js"
+        }
+    }
+
+    ## Test if tree-sitter is installed
+    if ($null -eq (Get-Command tree-sitter -ErrorAction Ignore)) {
+        Write-Verbose -Verbose "tree-sitter not found, installing..."
+        cargo install tree-sitter-cli --config .cargo/config.toml
+        if ($LASTEXITCODE -ne 0) {
+            throw "Failed to install tree-sitter-cli"
+        }
     }
 }
 
@@ -260,7 +281,7 @@ else {
 }
 
 if (!$SkipBuild) {
-    if ($architecture -ne 'Current') {
+    if ($architecture -ne 'Current' -and !$usingADO) {
         & $rustup target add --toolchain $channel $architecture
     }
 
@@ -309,6 +330,7 @@ if (!$SkipBuild) {
 
     # projects are in dependency order
     $projects = @(
+        ".",
         "tree-sitter-dscexpression",
         "tree-sitter-ssh-server-config",
         "security_context_lib",
@@ -345,9 +367,10 @@ if (!$SkipBuild) {
     $failed = $false
     foreach ($project in $projects) {
         ## Build format_json
-        Write-Host -ForegroundColor Cyan "Building $project ... for $architecture"
+        Write-Host -ForegroundColor Cyan "Building '$project' for $architecture"
         try {
             Push-Location "$PSScriptRoot/$project" -ErrorAction Stop
+            Write-Verbose -Verbose "Current directory is $(Get-Location)"
 
             # check if the project is either tree-sitter-dscexpression or tree-sitter-ssh-server-config
             if (($project -eq 'tree-sitter-dscexpression') -or ($project -eq 'tree-sitter-ssh-server-config')) {
@@ -357,12 +380,13 @@ if (!$SkipBuild) {
                 else {
                     if ($Audit) {
                         if ($null -eq (Get-Command cargo-audit -ErrorAction Ignore)) {
-                            cargo install cargo-audit --features=fix
+                            cargo install cargo-audit --features=fix --config .cargo/config.toml
                         }
 
                         cargo audit fix
                     }
 
+                    Write-Verbose -Verbose "Running build.ps1 for $project"
                     ./build.ps1
                 }
             }
@@ -389,7 +413,7 @@ if (!$SkipBuild) {
                     else {
                         if ($Audit) {
                             if ($null -eq (Get-Command cargo-audit -ErrorAction Ignore)) {
-                                cargo install cargo-audit --features=fix
+                                cargo install cargo-audit --features=fix --config .cargo/config.toml
                             }
 
                             cargo audit fix
@@ -399,12 +423,14 @@ if (!$SkipBuild) {
                             cargo clean
                         }
 
+                        Write-Verbose -Verbose "Building $project"
                         cargo build @flags
                     }
                 }
             }
 
-            if ($LASTEXITCODE -ne 0) {
+            if ($null -ne $LASTEXITCODE -and $LASTEXITCODE -ne 0) {
+                Write-Error "Last exit code is $LASTEXITCODE, build failed for '$project'"
                 $failed = $true
                 break
             }
@@ -453,7 +479,10 @@ if (!$SkipBuild) {
                     }
                 }
             }
-
+        } catch {
+            Write-Error "Failed to build $project : $($_ | Out-String)"
+            $failed = $true
+            break
         } finally {
             Pop-Location
         }
@@ -499,8 +528,6 @@ if (!$Clippy -and !$SkipBuild) {
 
 if ($Test) {
     $failed = $false
-
-    $usingADO = ($null -ne $env:TF_BUILD)
     $repository = 'PSGallery'
 
     if ($usingADO) {