Allow columns with default values to be excluded from the POCO (#388)

* allow missing default columns in poco

* add test

* query format

* add default column section to readme

* fix wording

Author: lucyzhang929

README.md

@@ -43,7 +43,7 @@ Azure SQL bindings for Azure Functions are supported for:
       - [ICollector<T>/IAsyncCollector<T>](#icollectortiasynccollectort)
       - [Array](#array)
       - [Single Row](#single-row)
-      - [Primary Keys and Identity Columns](#primary-keys-and-identity-columns)
+      - [Primary Key Special Cases](#primary-key-special-cases)
   - [Known Issues](#known-issues)
   - [Telemetry](#telemetry)
   - [Trademarks](#trademarks)
@@ -756,20 +756,26 @@ public static IActionResult Run(
 }
 ```
 
-#### Primary Keys and Identity Columns
+#### Primary Key Special Cases
 
 Normally Output Bindings require two things :
 
 1. The table being upserted to contains a Primary Key constraint (composed of one or more columns)
 2. Each of those columns must be present in the POCO object used in the attribute
 
-If either of these are false then an error will be thrown.
+Normally if either of these are false then an error will be thrown. Below are the situations in which this is not the case :
 
-This changes if one of the primary key columns is an identity column though. In that case there are two options based on how the function defines the output object:
+##### Identity Columns
+In the case where one of the primary key columns is an identity column, there are two options based on how the function defines the output object:
 
 1. If the identity column isn't included in the output object then a straight insert is always performed with the other column values. See [AddProductWithIdentityColumn](./samples/samples-csharp/OutputBindingSamples/AddProductWithIdentityColumn.cs) for an example.
 2. If the identity column is included (even if it's an optional nullable value) then a merge is performed similar to what happens when no identity column is present. This merge will either insert a new row or update an existing row based on the existence of a row that matches the primary keys (including the identity column). See [AddProductWithIdentityColumnIncluded](./samples/samples-csharp/OutputBindingSamples/AddProductWithIdentityColumnIncluded.cs) for an example.
 
+##### Columns with Default Values
+In the case where one of the primary key columns has a default value, there are also two options based on how the function defines the output object:
+1. If the column with a default value is not included in the output object, then a straight insert is always performed with the other values. See [AddProductWithDefaultPK](./samples/samples-csharp/OutputBindingSamples/AddProductWithDefaultPK.cs) for an example.
+2. If the column with a default value is included then a merge is performed similar to what happens when no default column is present. If there is a nullable column with a default value, then the provided column value in the output object will be upserted even if it is null.
+
 ## Known Issues
 
 - Output bindings against tables with columns of data types `NTEXT`, `TEXT`, or `IMAGE` are not supported and data upserts will fail. These types [will be removed](https://docs.microsoft.com/sql/t-sql/data-types/ntext-text-and-image-transact-sql) in a future version of SQL Server and are not compatible with the `OPENJSON` function used by this Azure Functions binding.

samples/samples-csharp/Common/Product.cs

@@ -25,4 +25,11 @@ public class ProductName
     {
         public string Name { get; set; }
     }
+
+    public class ProductWithDefaultPK
+    {
+        public string Name { get; set; }
+
+        public int Cost { get; set; }
+    }
 }

samples/samples-csharp/Database/Tables/ProductsWithDefaultPK.sql

@@ -0,0 +1,5 @@
+CREATE TABLE [ProductsWithDefaultPK] (
+	[ProductGuid] [uniqueidentifier] PRIMARY KEY NOT NULL DEFAULT(newsequentialid()),
+	[Name] [nvarchar](100) NULL,
+	[Cost] [int] NULL
+)

samples/samples-csharp/OutputBindingSamples/AddProductWithDefaultPK.cs

@@ -0,0 +1,31 @@
+// Copyright (c) Microsoft Corporation. All rights reserved.
+// Licensed under the MIT License. See License.txt in the project root for license information.
+
+using Microsoft.AspNetCore.Mvc;
+using Microsoft.Azure.WebJobs.Extensions.Http;
+using Microsoft.Azure.WebJobs.Extensions.Sql.Samples.Common;
+
+namespace Microsoft.Azure.WebJobs.Extensions.Sql.Samples.OutputBindingSamples
+{
+
+    public static class AddProductWithDefaultPK
+    {
+        /// <summary>
+        /// This shows an example of a SQL Output binding where the target table has a default primary key
+        /// of type uniqueidentifier and the column is not included in the output object. A new row will
+        /// be inserted and the uniqueidentifier will be generated by the engine.
+        /// </summary>
+        /// <param name="req">The original request that triggered the function</param>
+        /// <param name="product">The created ProductWithDefaultPK object</param>
+        /// <returns>The CreatedResult containing the new object that was inserted</returns>
+        [FunctionName(nameof(AddProductWithDefaultPK))]
+        public static IActionResult Run(
+            [HttpTrigger(AuthorizationLevel.Anonymous, "post", Route = "addproductwithdefaultpk")]
+            [FromBody] ProductWithDefaultPK product,
+            [Sql("dbo.ProductsWithDefaultPK", ConnectionStringSetting = "SqlConnectionString")] out ProductWithDefaultPK output)
+        {
+            output = product;
+            return new CreatedResult($"/api/addproductwithdefaultpk", output);
+        }
+    }
+}

samples/samples-js/AddProductWithDefaultPK/function.json

@@ -0,0 +1,27 @@
+{
+    "bindings": [
+      {
+        "authLevel": "function",
+        "name": "req",
+        "direction": "in",
+        "type": "httpTrigger",
+        "methods": [
+          "post"
+        ],
+        "route": "addproductwithdefaultpk"
+      },
+      {
+        "name": "$return",
+        "type": "http",
+        "direction": "out"
+      },
+      {
+        "name": "products",
+        "type": "sql",
+        "direction": "out",
+        "commandText": "[dbo].[ProductsWithDefaultPK]",
+        "connectionStringSetting": "SqlConnectionString"
+      }
+    ],
+    "disabled": false
+  }

samples/samples-js/AddProductWithDefaultPK/index.js

@@ -0,0 +1,16 @@
+// Copyright (c) Microsoft Corporation. All rights reserved.
+// Licensed under the MIT License. See License.txt in the project root for license information.
+
+// This shows an example of a SQL Output binding where the target table has a default primary key
+// of type uniqueidentifier and the column is not included in the output object. A new row will
+// be inserted and the uniqueidentifier will be generated by the engine.
+module.exports = async function (context, req) {
+    // Note that this expects the body to be a JSON object
+    // matching each of the columns in the table to upsert to.
+    context.bindings.products = req.body;
+
+    return {
+        status: 201,
+        body: req.body
+    };
+}

src/SqlAsyncCollector.cs

@@ -30,10 +30,13 @@ internal class PrimaryKey
 
         public readonly bool IsIdentity;
 
-        public PrimaryKey(string name, bool isIdentity)
+        public readonly bool HasDefault;
+
+        public PrimaryKey(string name, bool isIdentity, bool hasDefault)
         {
             this.Name = name;
             this.IsIdentity = isIdentity;
+            this.HasDefault = hasDefault;
         }
 
         public override string ToString()
@@ -49,6 +52,7 @@ internal class SqlAsyncCollector<T> : IAsyncCollector<T>, IDisposable
         private const string ColumnName = "COLUMN_NAME";
         private const string ColumnDefinition = "COLUMN_DEFINITION";
 
+        private const string HasDefault = "has_default";
         private const string IsIdentity = "is_identity";
         private const string CteName = "cte";
 
@@ -170,7 +174,7 @@ private async Task UpsertRowsAsync(IEnumerable<T> rows, SqlAttribute attribute,
                 if (tableInfo == null)
                 {
                     TelemetryInstance.TrackEvent(TelemetryEventName.TableInfoCacheMiss, props);
-                    // set the columnNames for supporting T as JObject since it doesn't have columns in the memeber info.
+                    // set the columnNames for supporting T as JObject since it doesn't have columns in the member info.
                     tableInfo = await TableInformation.RetrieveTableInformationAsync(connection, fullTableName, this._logger, GetColumnNamesFromItem(rows.First()));
                     var policy = new CacheItemPolicy
                     {
@@ -426,13 +430,20 @@ public static string GetPrimaryKeysQuery(SqlObject table)
             {
                 return $@"
                     SELECT
-                        {ColumnName}, c.is_identity
+                        ccu.{ColumnName},
+                        c.is_identity,
+                        case
+                            when isc.COLUMN_DEFAULT = NULL then 'false'
+                            else 'true'
+                        end as {HasDefault}
                     FROM
                         INFORMATION_SCHEMA.TABLE_CONSTRAINTS tc
                     INNER JOIN
                         INFORMATION_SCHEMA.CONSTRAINT_COLUMN_USAGE ccu ON ccu.CONSTRAINT_NAME = tc.CONSTRAINT_NAME AND ccu.TABLE_NAME = tc.TABLE_NAME
                     INNER JOIN
                         sys.columns c ON c.object_id = OBJECT_ID({table.QuotedFullName}) AND c.name = ccu.COLUMN_NAME
+                    INNER JOIN
+                        INFORMATION_SCHEMA.COLUMNS isc ON isc.TABLE_NAME = {table.QuotedName} AND isc.COLUMN_NAME = ccu.COLUMN_NAME
                     WHERE
                         tc.CONSTRAINT_TYPE = 'PRIMARY KEY'
                     and
@@ -464,15 +475,15 @@ INFORMATION_SCHEMA.COLUMNS c
                         c.TABLE_SCHEMA = {table.QuotedSchema}";
             }
 
-            public static string GetInsertQuery(SqlObject table)
+            public static string GetInsertQuery(SqlObject table, IEnumerable<string> bracketedColumnNamesFromItem)
             {
-                return $"INSERT INTO {table.BracketQuotedFullName} SELECT * FROM {CteName}";
+                return $"INSERT INTO {table.BracketQuotedFullName} ({string.Join(",", bracketedColumnNamesFromItem)}) SELECT * FROM {CteName}";
             }
 
             /// <summary>
             /// Generates reusable SQL query that will be part of every upsert command.
             /// </summary>
-            public static string GetMergeQuery(IList<PrimaryKey> primaryKeys, SqlObject table, StringComparison comparison, IEnumerable<string> columnNames)
+            public static string GetMergeQuery(IList<PrimaryKey> primaryKeys, SqlObject table, IEnumerable<string> bracketedColumnNamesFromItem)
             {
                 IList<string> bracketedPrimaryKeys = primaryKeys.Select(p => p.Name.AsBracketQuotedString()).ToList();
                 // Generate the ON part of the merge query (compares new data against existing data)
@@ -483,9 +494,6 @@ public static string GetMergeQuery(IList<PrimaryKey> primaryKeys, SqlObject tabl
                 }
 
                 // Generate the UPDATE part of the merge query (all columns that should be updated)
-                IEnumerable<string> bracketedColumnNamesFromItem = columnNames
-                    .Where(prop => !primaryKeys.Any(k => k.IsIdentity && string.Equals(k.Name, prop, comparison))) // Skip any identity columns, those should never be updated
-                    .Select(prop => prop.AsBracketQuotedString());
                 var columnMatchingQueryBuilder = new StringBuilder();
                 foreach (string column in bracketedColumnNamesFromItem)
                 {
@@ -604,7 +612,7 @@ public static async Task<TableInformation> RetrieveTableInformationAsync(SqlConn
                         while (await rdr.ReadAsync())
                         {
                             string columnName = caseSensitive ? rdr[ColumnName].ToString() : rdr[ColumnName].ToString().ToLowerInvariant();
-                            primaryKeys.Add(new PrimaryKey(columnName, bool.Parse(rdr[IsIdentity].ToString())));
+                            primaryKeys.Add(new PrimaryKey(columnName, bool.Parse(rdr[IsIdentity].ToString()), bool.Parse(rdr[HasDefault].ToString())));
                         }
                         primaryKeysSw.Stop();
                         TelemetryInstance.TrackDuration(TelemetryEventName.GetPrimaryKeys, primaryKeysSw.ElapsedMilliseconds, sqlConnProps);
@@ -634,20 +642,24 @@ public static async Task<TableInformation> RetrieveTableInformationAsync(SqlConn
                 IEnumerable<PrimaryKey> missingPrimaryKeysFromItem = primaryKeys
                     .Where(k => !primaryKeysFromObject.Contains(k.Name, comparer));
                 bool hasIdentityColumnPrimaryKeys = primaryKeys.Any(k => k.IsIdentity);
-                // If none of the primary keys are an identity column then we require that all primary keys be present in the POCO so we can
+                bool hasDefaultColumnPrimaryKeys = primaryKeys.Any(k => k.HasDefault);
+                // If none of the primary keys are an identity column or have a default value then we require that all primary keys be present in the POCO so we can
                 // generate the MERGE statement correctly
-                if (!hasIdentityColumnPrimaryKeys && missingPrimaryKeysFromItem.Any())
+                if (!hasIdentityColumnPrimaryKeys && !hasDefaultColumnPrimaryKeys && missingPrimaryKeysFromItem.Any())
                 {
                     string message = $"All primary keys for SQL table {table} need to be found in '{typeof(T)}.' Missing primary keys: [{string.Join(",", missingPrimaryKeysFromItem)}]";
                     var ex = new InvalidOperationException(message);
                     TelemetryInstance.TrackException(TelemetryErrorName.MissingPrimaryKeys, ex, sqlConnProps);
                     throw ex;
                 }
 
-                // If any identity columns aren't included in the object then we have to generate a basic insert since the merge statement expects all primary key
+                // If any identity columns or columns with default values aren't included in the object then we have to generate a basic insert since the merge statement expects all primary key
                 // columns to exist. (the merge statement can handle nullable columns though if those exist)
-                bool usingInsertQuery = hasIdentityColumnPrimaryKeys && missingPrimaryKeysFromItem.Any();
-                string query = usingInsertQuery ? GetInsertQuery(table) : GetMergeQuery(primaryKeys, table, comparison, columnNames);
+                bool usingInsertQuery = (hasIdentityColumnPrimaryKeys || hasDefaultColumnPrimaryKeys) && missingPrimaryKeysFromItem.Any();
+                IEnumerable<string> bracketedColumnNamesFromItem = columnNames
+                    .Where(prop => !primaryKeys.Any(k => k.IsIdentity && string.Equals(k.Name, prop, comparison))) // Skip any identity columns, those should never be updated
+                    .Select(prop => prop.AsBracketQuotedString());
+                string query = usingInsertQuery ? GetInsertQuery(table, bracketedColumnNamesFromItem) : GetMergeQuery(primaryKeys, table, bracketedColumnNamesFromItem);
 
                 tableInfoSw.Stop();
                 var durations = new Dictionary<TelemetryMeasureName, double>()

test/Integration/SqlOutputBindingIntegrationTests.cs

@@ -377,5 +377,25 @@ public void AddProductCaseSensitiveTest(SupportedLanguages lang)
             Assert.Equal("test", this.ExecuteScalar($"select Name from Products where ProductId={1}"));
             Assert.Equal(100, this.ExecuteScalar($"select cost from Products where ProductId={1}"));
         }
+
+        /// <summary>
+        /// Tests that a row is inserted successfully when the object is missing
+        /// the primary key column with a default value.
+        /// </summary>
+        [Theory]
+        [SqlInlineData()]
+        public void AddProductWithDefaultPKTest(SupportedLanguages lang)
+        {
+            this.StartFunctionHost(nameof(AddProductWithDefaultPK), lang);
+            var product = new Dictionary<string, string>()
+            {
+                { "name", "MyProduct" },
+                { "cost", "1" }
+            };
+            Assert.Equal(0, this.ExecuteScalar("SELECT COUNT(*) FROM dbo.ProductsWithDefaultPK"));
+            this.SendOutputPostRequest("addproductwithdefaultpk", JsonConvert.SerializeObject(product)).Wait();
+            this.SendOutputPostRequest("addproductwithdefaultpk", JsonConvert.SerializeObject(product)).Wait();
+            Assert.Equal(2, this.ExecuteScalar("SELECT COUNT(*) FROM dbo.ProductsWithDefaultPK"));
+        }
     }
 }