Completed filter support via streaming filter API

Added complete callback to facilitate long running or transforming filters like Git-LFS.

Added repository's working directory to callbacks to support extern process like Git-LFS, where knowledge of a repository's root maybe necissary but unobtainable in another fasion.

Removed unused NativeMethods

Author: J Wyman

LibGit2Sharp.Tests/FilterFixture.cs

@@ -8,9 +8,10 @@ namespace LibGit2Sharp.Tests
 {
     public class FilterFixture : BaseFixture
     {
-        private const int GitPassThrough = -30;
-
-        readonly Func<Stream, Stream, int> successCallback = (reader, writer) => 0;
+        readonly Action<Stream, Stream> successCallback = (reader, writer) =>
+        {
+            reader.CopyTo(writer);
+        };
 
         private const string FilterName = "the-filter";
         readonly List<FilterAttributeEntry> attributes = new List<FilterAttributeEntry> { new FilterAttributeEntry("test") };
@@ -19,7 +20,7 @@ public class FilterFixture : BaseFixture
         public void CanRegisterFilterWithSingleAttribute()
         {
             var filter = new EmptyFilter(FilterName, attributes);
-            Assert.Equal( attributes , filter.Attributes);
+            Assert.Equal(attributes, filter.Attributes);
         }
 
         [Fact]
@@ -56,10 +57,9 @@ public void SameFilterIsEqual()
         public void InitCallbackNotMadeWhenFilterNeverUsed()
         {
             bool called = false;
-            Func<int> initializeCallback = () =>
+            Action initializeCallback = () =>
             {
                 called = true;
-                return 0;
             };
 
             var filter = new FakeFilter(FilterName + 11, attributes,
@@ -78,10 +78,9 @@ public void InitCallbackNotMadeWhenFilterNeverUsed()
         public void InitCallbackMadeWhenUsingTheFilter()
         {
             bool called = false;
-            Func<int> initializeCallback = () =>
+            Action initializeCallback = () =>
             {
                 called = true;
-                return 0;
             };
 
             var filter = new FakeFilter(FilterName + 12, attributes,
@@ -108,10 +107,10 @@ public void WhenStagingFileApplyIsCalledWithCleanForCorrectPath()
             string repoPath = InitNewRepository();
             bool called = false;
 
-            Func<Stream, Stream, int> clean = (reader, writer) =>
+            Action<Stream, Stream> clean = (reader, writer) =>
             {
                 called = true;
-                return GitPassThrough;
+                reader.CopyTo(writer);
             };
             var filter = new FakeFilter(FilterName + 15, attributes, clean);
 
@@ -134,7 +133,7 @@ public void CleanFilterWritesOutputToObjectTree()
 
             string repoPath = InitNewRepository();
 
-            Func<Stream, Stream, int> cleanCallback = SubstitutionCipherFilter.RotateByThirteenPlaces;
+            Action<Stream, Stream> cleanCallback = SubstitutionCipherFilter.RotateByThirteenPlaces;
 
             var filter = new FakeFilter(FilterName + 16, attributes, cleanCallback);
 
@@ -154,7 +153,6 @@ public void CleanFilterWritesOutputToObjectTree()
             GlobalSettings.DeregisterFilter(filterRegistration);
         }
 
-
         [Fact]
         public void WhenCheckingOutAFileFileSmudgeWritesCorrectFileToWorkingDirectory()
         {
@@ -164,7 +162,7 @@ public void WhenCheckingOutAFileFileSmudgeWritesCorrectFileToWorkingDirectory()
             const string branchName = "branch";
             string repoPath = InitNewRepository();
 
-            Func<Stream, Stream, int> smudgeCallback = SubstitutionCipherFilter.RotateByThirteenPlaces;
+            Action<Stream, Stream> smudgeCallback = SubstitutionCipherFilter.RotateByThirteenPlaces;
 
             var filter = new FakeFilter(FilterName + 17, attributes, null, smudgeCallback);
             var filterRegistration = GlobalSettings.RegisterFilter(filter);
@@ -237,34 +235,55 @@ public EmptyFilter(string name, IEnumerable<FilterAttributeEntry> attributes)
 
         class FakeFilter : Filter
         {
-            private readonly Func<Stream, Stream, int> cleanCallback;
-            private readonly Func<Stream, Stream, int> smudgeCallback;
-            private readonly Func<int> initCallback;
+            private readonly Action<Stream, Stream> cleanCallback;
+            private readonly Action<Stream, Stream> smudgeCallback;
+            private readonly Action initCallback;
 
             public FakeFilter(string name, IEnumerable<FilterAttributeEntry> attributes,
-                Func<Stream, Stream, int> cleanCallback = null,
-                Func<Stream, Stream, int> smudgeCallback = null,
-                Func<int> initCallback = null)
+                Action<Stream, Stream> cleanCallback = null,
+                Action<Stream, Stream> smudgeCallback = null,
+                Action initCallback = null)
                 : base(name, attributes)
             {
                 this.cleanCallback = cleanCallback;
                 this.smudgeCallback = smudgeCallback;
                 this.initCallback = initCallback;
             }
 
-            protected override int Clean(string path, Stream input, Stream output)
+            protected override void Clean(string path, string root, Stream input, Stream output)
             {
-                return cleanCallback != null ? cleanCallback(input, output) : base.Clean(path, input, output);
+                if (cleanCallback == null)
+                {
+                    base.Clean(path, root, input, output);
+                }
+                else
+                {
+                    cleanCallback(input, output);
+                }
             }
 
-            protected override int Smudge(string path, Stream input, Stream output)
+            protected override void Smudge(string path, string root, Stream input, Stream output)
             {
-                return smudgeCallback != null ? smudgeCallback(input, output) : base.Smudge(path, input, output);
+                if (smudgeCallback == null)
+                {
+                    base.Smudge(path, root, input, output);
+                }
+                else
+                {
+                    smudgeCallback(input, output);
+                }
             }
 
-            protected override int Initialize()
+            protected override void Initialize()
             {
-                return initCallback != null ? initCallback() : base.Initialize();
+                if (initCallback == null)
+                {
+                    base.Initialize();
+                }
+                else
+                {
+                    initCallback();
+                }
             }
         }
     }

LibGit2Sharp.Tests/FilterSubstitutionCipherFixture.cs

@@ -15,7 +15,7 @@ public void SmugdeIsNotCalledForFileWhichDoesNotMatchAnAttributeEntry()
             const string decodedInput = "This is a substitution cipher";
             const string encodedInput = "Guvf vf n fhofgvghgvba pvcure";
 
-            var attributes = new List<FilterAttributeEntry> { new FilterAttributeEntry("filter=rot13") };
+            var attributes = new List<FilterAttributeEntry> { new FilterAttributeEntry("rot13") };
             var filter = new SubstitutionCipherFilter("cipher-filter", attributes);
             var filterRegistration = GlobalSettings.RegisterFilter(filter);
 
@@ -55,7 +55,7 @@ public void CorrectlyEncodesAndDecodesInput()
             const string decodedInput = "This is a substitution cipher";
             const string encodedInput = "Guvf vf n fhofgvghgvba pvcure";
 
-            var attributes = new List<FilterAttributeEntry> { new FilterAttributeEntry("filter=rot13") };
+            var attributes = new List<FilterAttributeEntry> { new FilterAttributeEntry("rot13") };
             var filter = new SubstitutionCipherFilter("cipher-filter", attributes);
             var filterRegistration = GlobalSettings.RegisterFilter(filter);
 
@@ -94,9 +94,9 @@ public void CorrectlyEncodesAndDecodesInput()
         [InlineData("*.txt", ".txt", 1, 0)]
         public void WhenStagedFileDoesNotMatchPathSpecFileIsNotFiltered(string pathSpec, string fileExtension, int cleanCount, int smudgeCount)
         {
-            const string filterName = "filter=rot13";
+            const string filterName = "rot13";
             const string decodedInput = "This is a substitution cipher";
-            string attributeFileEntry = string.Format("{0} {1}", pathSpec, filterName);
+            string attributeFileEntry = string.Format("{0} filter={1}", pathSpec, filterName);
 
             var filterForAttributes = new List<FilterAttributeEntry> { new FilterAttributeEntry(filterName) };
             var filter = new SubstitutionCipherFilter("cipher-filter", filterForAttributes);
@@ -122,15 +122,13 @@ public void WhenStagedFileDoesNotMatchPathSpecFileIsNotFiltered(string pathSpec,
         }
 
         [Theory]
-        [InlineData("filter=rot13", "*.txt filter=rot13", 1)]
-        [InlineData("filter=rot13", "*.txt filter=fake", 0)]
-        [InlineData("filter=rot13", "*.bat filter=rot13", 0)]
         [InlineData("rot13", "*.txt filter=rot13", 1)]
         [InlineData("rot13", "*.txt filter=fake", 0)]
+        [InlineData("rot13", "*.bat filter=rot13", 0)]
+        [InlineData("rot13", "*.txt filter=fake", 0)]
         [InlineData("fake", "*.txt filter=fake", 1)]
-        [InlineData("filter=fake", "*.txt filter=fake", 1)]
-        [InlineData("filter=fake", "*.bat filter=fake", 0)]
-        [InlineData("filter=rot13", "*.txt filter=rot13 -crlf", 1)]
+        [InlineData("fake", "*.bat filter=fake", 0)]
+        [InlineData("rot13", "*.txt filter=rot13 -crlf", 1)]
         public void CleanIsCalledIfAttributeEntryMatches(string filterAttribute, string attributeEntry, int cleanCount)
         {
             const string decodedInput = "This is a substitution cipher";
@@ -158,12 +156,10 @@ public void CleanIsCalledIfAttributeEntryMatches(string filterAttribute, string
         }
 
         [Theory]
-        [InlineData("filter=rot13", "*.txt filter=rot13", 1)]
-        [InlineData("filter=rot13", "*.txt filter=fake", 0)]
-        [InlineData("filter=rot13", "*.bat filter=rot13", 0)]
+
         [InlineData("rot13", "*.txt filter=rot13", 1)]
         [InlineData("rot13", "*.txt filter=fake", 0)]
-        [InlineData("filter=rot13", "*.txt filter=rot13 -crlf", 1)]
+        [InlineData("rot13", "*.txt filter=rot13 -crlf", 1)]
         public void SmudgeIsCalledIfAttributeEntryMatches(string filterAttribute, string attributeEntry, int smudgeCount)
         {
             const string decodedInput = "This is a substitution cipher";

LibGit2Sharp.Tests/LibGit2Sharp.Tests.csproj

@@ -163,4 +163,4 @@
   <Target Name="AfterBuild">
   </Target>
   -->
-</Project>
+</Project>

LibGit2Sharp.Tests/TestHelpers/SubstitutionCipherFilter.cs

@@ -13,19 +13,19 @@ public SubstitutionCipherFilter(string name, IEnumerable<FilterAttributeEntry> a
         {
         }
 
-        protected override int Clean(string path, Stream input, Stream output)
+        protected override void Clean(string path, string root, Stream input, Stream output)
         {
             CleanCalledCount++;
-            return RotateByThirteenPlaces(input, output);
+            RotateByThirteenPlaces(input, output);
         }
 
-        protected override int Smudge(string path, Stream input, Stream output)
+        protected override void Smudge(string path, string root, Stream input, Stream output)
         {
             SmudgeCalledCount++;
-            return RotateByThirteenPlaces(input, output);
+            RotateByThirteenPlaces(input, output);
         }
 
-        public static int RotateByThirteenPlaces(Stream input, Stream output)
+        public static void RotateByThirteenPlaces(Stream input, Stream output)
         {
             int value;
 
@@ -42,8 +42,6 @@ public static int RotateByThirteenPlaces(Stream input, Stream output)
 
                 output.WriteByte((byte)value);
             }
-
-            return 0;
         }
     }
 }

LibGit2Sharp/Core/Ensure.cs

@@ -87,6 +87,33 @@ public static void ArgumentDoesNotContainZeroByte(string argumentValue, string a
                     "Zero bytes ('\\0') are not allowed. A zero byte has been found at position {0}.", zeroPos), argumentName);
         }
 
+        /// <summary>
+        /// Checks an argument to ensure it isn't a IntPtr.Zero (aka null).
+        /// </summary>
+        /// <param name="argumentValue">The argument value to check.</param>
+        /// <param name="argumentName">The name of the argument.</param>
+        public static void ArgumentNotZeroIntPtr(IntPtr argumentValue, string argumentName)
+        {
+            if (argumentValue == IntPtr.Zero)
+            {
+                throw new ArgumentNullException(argumentName);
+            }
+        }
+
+        /// <summary>
+        /// Checks a pointer argument to ensure it is the expected pointer value.
+        /// </summary>
+        /// <param name="argumentValue">The argument value to check.</param>
+        /// <param name="expectedValue">The expected value.</param>
+        /// <param name="argumentName">The name of the argument.</param>
+        public static void ArgumentIsExpectedIntPtr(IntPtr argumentValue, IntPtr expectedValue, string argumentName)
+        {
+            if (argumentValue != expectedValue)
+            {
+                throw new ArgumentException("Unexpected IntPtr value", argumentName);
+            }
+        }
+
         private static readonly Dictionary<GitErrorCode, Func<string, GitErrorCode, GitErrorCategory, LibGit2SharpException>>
             GitErrorsToLibGit2SharpExceptions =
                 new Dictionary<GitErrorCode, Func<string, GitErrorCode, GitErrorCategory, LibGit2SharpException>>

LibGit2Sharp/Core/GitFilter.cs

@@ -24,6 +24,9 @@ internal class GitFilter
         [MarshalAs(UnmanagedType.FunctionPtr)]
         public git_filter_apply_fn apply;
 
+        [MarshalAs(UnmanagedType.FunctionPtr)]
+        public git_filter_stream_fn stream;
+
         [MarshalAs(UnmanagedType.FunctionPtr)]
         public git_filter_cleanup_fn cleanup;
 
@@ -81,6 +84,9 @@ public delegate int git_filter_check_fn(
         public delegate int git_filter_apply_fn(
             GitFilter gitFilter, IntPtr payload, IntPtr gitBufTo, IntPtr gitBufFrom, IntPtr filterSource);
 
+        public delegate int git_filter_stream_fn(
+            out IntPtr git_writestream_out, GitFilter self, IntPtr payload, IntPtr filterSource, IntPtr git_writestream_next);
+
         /// <summary>
         /// Callback to clean up after filtering has been applied. Specified as `filter.cleanup`, this is an optional callback invoked
         /// after the filter has been applied.  If the `check` or `apply` callbacks allocated a `payload` 

LibGit2Sharp/Core/GitWriteStream.cs

@@ -0,0 +1,22 @@
+using System;
+using System.Runtime.InteropServices;
+
+namespace LibGit2Sharp.Core
+{
+    [StructLayout(LayoutKind.Sequential)]
+    internal class GitWriteStream
+    {
+        [MarshalAs(UnmanagedType.FunctionPtr)]
+        public write_fn write;
+
+        [MarshalAs(UnmanagedType.FunctionPtr)]
+        public close_fn close;
+
+        [MarshalAs(UnmanagedType.FunctionPtr)]
+        public free_fn free;        
+
+        public delegate int write_fn(IntPtr stream, IntPtr buffer, UIntPtr len);
+        public delegate int close_fn(IntPtr stream);
+        public delegate void free_fn(IntPtr stream);
+    }
+}

LibGit2Sharp/Core/NativeMethods.cs

@@ -191,12 +191,6 @@ internal static extern int git_branch_remote_name(
             RepositorySafeHandle repo,
             [MarshalAs(UnmanagedType.CustomMarshaler, MarshalCookie = UniqueId.UniqueIdentifier, MarshalTypeRef = typeof(StrictUtf8Marshaler))] string canonical_branch_name);
 
-        [DllImport(libgit2)]
-        internal static extern int git_buf_grow(IntPtr buffer, UIntPtr targetSize);
-
-        [DllImport(libgit2)]
-        internal static extern int git_buf_put(IntPtr buffer, IntPtr data, UIntPtr len);
-
         [DllImport(libgit2)]
         internal static extern int git_remote_rename(
             ref GitStrArray problems,
@@ -523,9 +517,6 @@ internal static extern int git_filter_unregister(
         [DllImport(libgit2)]
         internal static extern int git_filter_source_mode(IntPtr source);
 
-        [DllImport(libgit2)]
-        internal static extern void git_filter_free(IntPtr filterSafeHandle);
-
         [DllImport(libgit2)]
         internal static extern int git_libgit2_features();
 
@@ -1304,6 +1295,10 @@ internal static extern int git_repository_state(
         [return: MarshalAs(UnmanagedType.CustomMarshaler, MarshalCookie = UniqueId.UniqueIdentifier, MarshalTypeRef = typeof(LaxFilePathNoCleanupMarshaler))]
         internal static extern FilePath git_repository_workdir(RepositorySafeHandle repository);
 
+        [DllImport(libgit2)]
+        [return: MarshalAs(UnmanagedType.CustomMarshaler, MarshalCookie = UniqueId.UniqueIdentifier, MarshalTypeRef = typeof(LaxFilePathNoCleanupMarshaler))]
+        internal static extern FilePath git_repository_workdir(IntPtr repository);
+
         [DllImport(libgit2)]
         internal static extern int git_repository_new(out RepositorySafeHandle repo);