Merge branch 'main' into unit

Author: NiklasGustafsson

RELEASENOTES.md

@@ -25,6 +25,10 @@ __Issues fixed__:
 #1400 There may be an error in torchvision.transforms.GaussianBlur<br/>
 #1402 diagonal() has incorrect default<br/>
 
+__API Changes__:
+
+ #1382: Add support for torch.nn.functional.normalize<br/>
+
 # NuGet Version 0.103.1
 
 __Breaking Changes__:

TorchSharp.sln

@@ -36,7 +36,7 @@ Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "TorchSharp", "TorchSharp",
 EndProject
 Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "LibTorchSharp", "bin\obj\x64.Debug\Native\LibTorchSharp\LibTorchSharp.vcxproj", "{CAD9DB7F-3223-3324-884D-FA2381593DA7}"
 EndProject
-Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "LibTorchSharp", "bin\obj\x64.Release\Native\LibTorchSharp\LibTorchSharp.vcxproj", "{E4C0DBEE-0815-311B-9065-137BB50BD793}"
+Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "LibTorchSharp", "bin\obj\x64.Release\Native\LibTorchSharp\LibTorchSharp.vcxproj", "{BB811429-0DF1-3D22-B664-09C2F5A9E0AB}"
 EndProject
 Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "Native-Debug", "Native-Debug", "{CF2C1A9E-3A8A-4329-8A6E-7880C15AAC3D}"
 	ProjectSection(SolutionItems) = preProject
@@ -181,7 +181,7 @@ Global
 		{42B45168-476D-4BFA-87B8-81A34E6295CD} = {09EADF06-BE25-4228-AB53-95AE3E15B530}
 		{567456AD-B026-4CB6-B98D-4FC930C90223} = {D3D38B03-B557-484D-8348-8BADEE4DF592}
 		{CAD9DB7F-3223-3324-884D-FA2381593DA7} = {CF2C1A9E-3A8A-4329-8A6E-7880C15AAC3D}
-		{E4C0DBEE-0815-311B-9065-137BB50BD793} = {4DB9E84D-324C-408F-87A6-246E86205540}
+		{BB811429-0DF1-3D22-B664-09C2F5A9E0AB} = {4DB9E84D-324C-408F-87A6-246E86205540}
 		{CF2C1A9E-3A8A-4329-8A6E-7880C15AAC3D} = {09EADF06-BE25-4228-AB53-95AE3E15B530}
 		{D8C60CD8-8429-45F2-A755-47B6CD10FDF8} = {09EADF06-BE25-4228-AB53-95AE3E15B530}
 		{4DB9E84D-324C-408F-87A6-246E86205540} = {CF2C1A9E-3A8A-4329-8A6E-7880C15AAC3D}

src/Native/LibTorchSharp/THSNN.h

@@ -39,6 +39,7 @@ EXPORT_API(NNModule) THSNN_custom_module(const char* name, Tensor(*forward)(Tens
 
 // Normalization
 
+EXPORT_API(Tensor)   THSNN_normalize(const Tensor input, const double p, const int64_t dim, const double eps);
 EXPORT_API(Tensor)   THSNN_batch_norm(const Tensor input, const Tensor running_mean, const Tensor running_var, const Tensor weight, const Tensor bias, const bool training, const double momentum, const double eps);
 EXPORT_API(Tensor)   THSNN_group_norm(const Tensor input, int64_t num_groups, const Tensor weight, const Tensor bias, const double eps);
 EXPORT_API(Tensor)   THSNN_instance_norm(const Tensor input, const Tensor running_mean, const Tensor running_var, const Tensor weight, const Tensor bias, const bool use_input_stats, const double momentum, const double eps);

src/Native/LibTorchSharp/THSNormalization.cpp

@@ -13,6 +13,15 @@ Tensor THSNN_batch_norm(const Tensor input, Tensor running_mean, const Tensor ru
     CATCH_TENSOR(torch::batch_norm(*input, w, b, rm, rv, training, momentum, eps, false));
 }
 
+Tensor THSNN_normalize(const Tensor input, const double p, const int64_t dim, const double eps)
+{
+    auto opts = torch::nn::functional::NormalizeFuncOptions()
+        .p(p)
+        .dim(dim)
+        .eps(eps);
+    CATCH_TENSOR(torch::nn::functional::normalize(*input, opts));
+}
+
 Tensor THSNN_group_norm(const Tensor input, const int64_t num_groups, const Tensor weight, const Tensor bias, const double eps)
 {
     auto opts = torch::nn::functional::GroupNormFuncOptions(num_groups)

src/TorchSharp/NN/Normalization/Functional.cs

@@ -2,6 +2,7 @@
 using System;
 using static TorchSharp.PInvoke.NativeMethods;
 
+#nullable enable
 namespace TorchSharp
 {
     public static partial class torch
@@ -10,10 +11,27 @@ public static partial class nn
         {
             public static partial class functional
             {
+                /// <summary>
+                /// Perform normalization of inputs over specified dimension.
+                /// </summary>
+                /// <param name="input">Input tensor of any shape.</param>
+                /// <param name="p">the exponent value in the norm formulation</param>
+                /// <param name="dim">the dimension to reduce</param>
+                /// <param name="eps">small value to avoid division by zero</param>
+                public static Tensor normalize(Tensor input, double p = 2.0, long dim = 1L, double eps = 1e-12)
+                {
+                    var res = THSNN_normalize(
+                        input.Handle,
+                        p, dim, eps);
+                    if (res == IntPtr.Zero)
+                        torch.CheckForErrors();
+                    return new Tensor(res);
+                }
+
                 /// <summary>
                 /// Applies Batch Normalization for each channel across a batch of data.
                 /// </summary>
-                public static Tensor batch_norm(Tensor input, Tensor running_mean, Tensor running_var, Tensor weight = null, Tensor bias = null, bool training = false, double momentum = 0.1, double eps = 1e-5)
+                public static Tensor batch_norm(Tensor input, Tensor running_mean, Tensor running_var, Tensor? weight = null, Tensor? bias = null, bool training = false, double momentum = 0.1, double eps = 1e-5)
                 {
                     var res = THSNN_batch_norm(
                         input.Handle,
@@ -31,7 +49,7 @@ public static Tensor batch_norm(Tensor input, Tensor running_mean, Tensor runnin
                 /// <summary>
                 /// Applies Group Normalization for last certain number of dimensions.
                 /// </summary>
-                public static Tensor group_norm(Tensor input, long num_groups, Tensor weight = null, Tensor bias = null, double eps = 1e-5)
+                public static Tensor group_norm(Tensor input, long num_groups, Tensor? weight = null, Tensor? bias = null, double eps = 1e-5)
                 {
                     var res = THSNN_group_norm(
                         input.Handle,
@@ -47,7 +65,7 @@ public static Tensor group_norm(Tensor input, long num_groups, Tensor weight = n
                 /// <summary>
                 /// Applies Instance Normalization for each channel in each data sample in a batch.
                 /// </summary>
-                public static Tensor instance_norm(Tensor input, Tensor running_mean = null, Tensor running_var = null, Tensor weight = null, Tensor bias = null, bool use_input_stats = true, double momentum = 0.1, double eps = 1e-5)
+                public static Tensor instance_norm(Tensor input, Tensor? running_mean = null, Tensor? running_var = null, Tensor? weight = null, Tensor? bias = null, bool use_input_stats = true, double momentum = 0.1, double eps = 1e-5)
                 {
                     var res = THSNN_instance_norm(
                         input.Handle,
@@ -65,7 +83,7 @@ public static Tensor instance_norm(Tensor input, Tensor running_mean = null, Ten
                 /// <summary>
                 /// Applies Layer Normalization for last certain number of dimensions.
                 /// </summary>
-                public static Tensor layer_norm(Tensor input, long[] normalized_shape, Tensor weight = null, Tensor bias = null, double eps = 1e-5)
+                public static Tensor layer_norm(Tensor input, long[] normalized_shape, Tensor? weight = null, Tensor? bias = null, double eps = 1e-5)
                 {
                     IntPtr res;
                     unsafe {

src/TorchSharp/PInvoke/LibTorchSharp.THSNN.cs

@@ -512,6 +512,9 @@ internal static extern IntPtr THSNN_custom_module(
         [DllImport("LibTorchSharp")]
         internal static extern IntPtr THSNN_Unflatten_ctor(long dim, IntPtr shape, long shape_len, out IntPtr pBoxedModule);
 
+        [DllImport("LibTorchSharp")]
+        internal static extern IntPtr THSNN_normalize(IntPtr input, double p, long dim, double eps);
+
         [DllImport("LibTorchSharp")]
         internal static extern IntPtr THSNN_batch_norm(IntPtr input, IntPtr running_mean, IntPtr running_var, IntPtr weight, IntPtr bias, [MarshalAs(UnmanagedType.U1)] bool training, double momentum, double eps);
 

test/TorchSharpTest/NN.cs

@@ -1,4 +1,4 @@
-// Copyright (c) .NET Foundation and Contributors.  All Rights Reserved.  See LICENSE in the project root for license information.
+ // Copyright (c) .NET Foundation and Contributors.  All Rights Reserved.  See LICENSE in the project root for license information.
 using System;
 using System.Linq;
 using System.Runtime.InteropServices;
@@ -5024,6 +5024,36 @@ private Tensor NormalizeTensor(Tensor x, Tensor x_mean, Tensor x_var, double eps
             return (x - x_mean) / torch.sqrt(eps + x_var);
         }
 
+        [Fact]
+        public void TestNormalizeFunc()
+        {
+            foreach (var device in TestUtils.AvailableDevices()) {
+                var x = torch.from_array(new double[]
+                    { -1.0786,  0.3455,  1.2929,  0.5030,
+                      -0.2930,  1.0420, -0.1082, -0.2943,
+                      -0.3989, -0.8311,  0.7103, -1.5878,
+                       0.6331,  1.0106,  0.5128, -2.2565,
+                       1.2044, -0.6916, -0.1242,  0.6808,
+                       0.1672,  0.1105, -1.7364,  0.0669
+                    }).reshape(3,2,4);
+                var y = torch.nn.functional.normalize(x);
+                Assert.Equal(x.shape, y.shape);
+                Assert.Equal(x.device_type, y.device_type);
+
+                var expected = torch.from_array(new double[]
+                    { -0.9650,  0.3147,  0.9965,  0.8631,
+                      -0.2621,  0.9492, -0.0834, -0.5050,
+                      -0.5331, -0.6352,  0.8108, -0.5755,
+                       0.8460,  0.7724,  0.5853, -0.8178,
+                       0.9905, -0.9875, -0.0713,  0.9952,
+                       0.1375,  0.1577, -0.9975,  0.0978
+                    }).reshape(3, 2, 4);
+
+
+                Assert.True(y.allclose(expected, rtol: 0.005, atol: 0.005));
+            }
+        }
+
         [Fact]
         public void TestBatchNormFunc()
         {