Copy latest sql dll to extension bundle in pipeline (#395)

* cp sql dll to extension bundle

* remove quotes + add windows condition

* fix path

* add bin to path + add linux

* use source folder

* use preview path

* add overwrite and ignore mkdir errors

* fix target folder path

* remove ignoreMakeDirErrors

* add wait for case sensitivity test

* use one task

* set cache timeout to 0

* fix target path

* pr comments + add timeout var linux

* set cache to 0 for one test

Author: lucyzhang929

builds/azure-pipelines/template-steps-build-test.yml

@@ -32,6 +32,10 @@ steps:
   displayName: 'Set npm installation path for Windows'
   condition: and(succeeded(), eq(variables['Agent.OS'], 'Windows_NT'))
 
+- bash: echo "##vso[task.setvariable variable=azureFunctionsExtensionBundlePath]$(func GetExtensionBundlePath)"
+  displayName: 'Set Azure Functions extension bundle path'
+  workingDirectory: $(Build.SourcesDirectory)/samples/samples-js
+
 - task: DockerInstaller@0
   displayName: Docker Installer
   inputs:
@@ -67,6 +71,14 @@ steps:
     projects: '${{ parameters.solution }}'
     arguments: '--configuration ${{ parameters.configuration }} -p:GeneratePackageOnBuild=false -p:Version=${{ parameters.binariesVersion }}'
 
+- task: CopyFiles@2
+  displayName: 'Copy Sql extension dll to Azure Functions extension bundle'
+  inputs:
+    sourceFolder: $(Build.SourcesDirectory)/src/bin/${{ parameters.configuration }}/netstandard2.0
+    contents: Microsoft.Azure.WebJobs.Extensions.Sql.dll
+    targetFolder: $(azureFunctionsExtensionBundlePath)/bin
+    overWrite: true
+
 - script: |
     npm install
     npm run lint

src/SqlAsyncCollector.cs

@@ -2,6 +2,7 @@
 // Licensed under the MIT License. See License.txt in the project root for license information.
 
 using System;
+using System.Globalization;
 using System.Collections.Generic;
 using System.Data;
 using System.Linq;
@@ -58,6 +59,8 @@ internal class SqlAsyncCollector<T> : IAsyncCollector<T>, IDisposable
 
         private const string Collation = "Collation";
 
+        private const int AZ_FUNC_TABLE_INFO_CACHE_TIMEOUT_MINUTES = 10;
+
         private readonly IConfiguration _configuration;
         private readonly SqlAttribute _attribute;
         private readonly ILogger _logger;
@@ -171,15 +174,29 @@ private async Task UpsertRowsAsync(IEnumerable<T> rows, SqlAttribute attribute,
                 ObjectCache cachedTables = MemoryCache.Default;
                 var tableInfo = cachedTables[cacheKey] as TableInformation;
 
+                int timeout = AZ_FUNC_TABLE_INFO_CACHE_TIMEOUT_MINUTES;
+                string timeoutEnvVar = Environment.GetEnvironmentVariable("AZ_FUNC_TABLE_INFO_CACHE_TIMEOUT_MINUTES");
+                if (!string.IsNullOrEmpty(timeoutEnvVar))
+                {
+                    if (int.TryParse(timeoutEnvVar, NumberStyles.Integer, CultureInfo.InvariantCulture, out timeout))
+                    {
+                        this._logger.LogDebugWithThreadId($"Overriding default table info cache timeout with new value {timeout}");
+                    }
+                    else
+                    {
+                        timeout = AZ_FUNC_TABLE_INFO_CACHE_TIMEOUT_MINUTES;
+                    }
+                }
+
                 if (tableInfo == null)
                 {
                     TelemetryInstance.TrackEvent(TelemetryEventName.TableInfoCacheMiss, props);
                     // set the columnNames for supporting T as JObject since it doesn't have columns in the member info.
                     tableInfo = await TableInformation.RetrieveTableInformationAsync(connection, fullTableName, this._logger, GetColumnNamesFromItem(rows.First()));
                     var policy = new CacheItemPolicy
                     {
-                        // Re-look up the primary key(s) after 10 minutes (they should not change very often!)
-                        AbsoluteExpiration = DateTimeOffset.Now.AddMinutes(10)
+                        // Re-look up the primary key(s) after timeout (default timeout is 10 minutes)
+                        AbsoluteExpiration = DateTimeOffset.Now.AddMinutes(timeout)
                     };
 
                     cachedTables.Set(cacheKey, tableInfo, policy);

test/Integration/IntegrationTestBase.cs

@@ -176,7 +176,7 @@ protected void StartAzurite()
         /// - The functionName is different than its route.<br/>
         /// - You can start multiple functions by passing in a space-separated list of function names.<br/>
         /// </remarks>
-        protected void StartFunctionHost(string functionName, SupportedLanguages language, bool useTestFolder = false, DataReceivedEventHandler customOutputHandler = null)
+        protected void StartFunctionHost(string functionName, SupportedLanguages language, bool useTestFolder = false, DataReceivedEventHandler customOutputHandler = null, Dictionary<string, string> environmentVariables = null)
         {
             string workingDirectory = language == SupportedLanguages.CSharp && useTestFolder ? GetPathToBin() : Path.Combine(GetPathToBin(), "SqlExtensionSamples", Enum.GetName(typeof(SupportedLanguages), language));
             if (!Directory.Exists(workingDirectory))
@@ -196,6 +196,14 @@ protected void StartFunctionHost(string functionName, SupportedLanguages languag
                 UseShellExecute = false
             };
 
+            if (environmentVariables != null)
+            {
+                foreach (KeyValuePair<string, string> variable in environmentVariables)
+                {
+                    startInfo.EnvironmentVariables[variable.Key] = variable.Value;
+                }
+            }
+
             // Always disable telemetry during test runs
             startInfo.EnvironmentVariables[TelemetryOptoutEnvVar] = "1";
 

test/Integration/SqlOutputBindingIntegrationTests.cs

@@ -352,10 +352,15 @@ public void AddProductWithIdentity_MissingPrimaryColumn(SupportedLanguages lang)
         [SqlInlineData()]
         public void AddProductCaseSensitiveTest(SupportedLanguages lang)
         {
-            this.StartFunctionHost(nameof(AddProductParams), lang);
+            // Set table info cache timeout to 0 minutes so that new collation gets picked up
+            var environmentVariables = new Dictionary<string, string>()
+            {
+                { "AZ_FUNC_TABLE_INFO_CACHE_TIMEOUT_MINUTES", "0" }
+            };
+            this.StartFunctionHost(nameof(AddProductParams), lang, false, null, environmentVariables);
 
             // Change database collation to case sensitive
-            this.ExecuteNonQuery($"ALTER DATABASE {this.DatabaseName} COLLATE Latin1_General_CS_AS");
+            this.ExecuteNonQuery($"ALTER DATABASE {this.DatabaseName} SET Single_User WITH ROLLBACK IMMEDIATE; ALTER DATABASE {this.DatabaseName} COLLATE Latin1_General_CS_AS; ALTER DATABASE {this.DatabaseName} SET Multi_User;");
 
             var query = new Dictionary<string, string>()
             {
@@ -369,7 +374,7 @@ public void AddProductCaseSensitiveTest(SupportedLanguages lang)
             Assert.Throws<AggregateException>(() => this.SendOutputGetRequest("addproduct-params", query).Wait());
 
             // Change database collation back to case insensitive
-            this.ExecuteNonQuery($"ALTER DATABASE {this.DatabaseName} COLLATE Latin1_General_CI_AS");
+            this.ExecuteNonQuery($"ALTER DATABASE {this.DatabaseName} SET Single_User WITH ROLLBACK IMMEDIATE; ALTER DATABASE {this.DatabaseName} COLLATE Latin1_General_CI_AS; ALTER DATABASE {this.DatabaseName} SET Multi_User;");
 
             this.SendOutputGetRequest("addproduct-params", query).Wait();