From 14ab70eae9346ae53707cdbf57782af0e7971cd3 Mon Sep 17 00:00:00 2001 From: Keith Dahlby Date: Fri, 26 Jul 2013 12:54:43 -0500 Subject: [PATCH 1/3] Add optional encoding to Touch() --- LibGit2Sharp.Tests/TestHelpers/BaseFixture.cs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/LibGit2Sharp.Tests/TestHelpers/BaseFixture.cs b/LibGit2Sharp.Tests/TestHelpers/BaseFixture.cs index 6dc0c76ad..c3881e015 100644 --- a/LibGit2Sharp.Tests/TestHelpers/BaseFixture.cs +++ b/LibGit2Sharp.Tests/TestHelpers/BaseFixture.cs @@ -218,7 +218,7 @@ private static RepositoryOptions BuildFakeRepositoryOptions(SelfCleaningDirector }; } - protected static string Touch(string parent, string file, string content = null) + protected static string Touch(string parent, string file, string content = null, Encoding encoding = null) { string filePath = Path.Combine(parent, file); string dir = Path.GetDirectoryName(filePath); @@ -226,7 +226,7 @@ protected static string Touch(string parent, string file, string content = null) Directory.CreateDirectory(dir); - File.WriteAllText(filePath, content ?? string.Empty, Encoding.ASCII); + File.WriteAllText(filePath, content ?? string.Empty, encoding ?? Encoding.ASCII); return filePath; } From a50756e30597d2fb66d5ff74078b72fb96aeee8b Mon Sep 17 00:00:00 2001 From: Keith Dahlby Date: Sun, 4 Aug 2013 21:51:36 -0500 Subject: [PATCH 2/3] Introduce Blob.ContentAsText() --- LibGit2Sharp.Tests/AttributesFixture.cs | 2 +- LibGit2Sharp.Tests/BlobFixture.cs | 40 +++++++++++++++++++-- LibGit2Sharp.Tests/CommitFixture.cs | 4 +-- LibGit2Sharp.Tests/ObjectDatabaseFixture.cs | 2 +- LibGit2Sharp/BlobExtensions.cs | 25 +++++++++++-- 5 files changed, 64 insertions(+), 9 deletions(-) diff --git a/LibGit2Sharp.Tests/AttributesFixture.cs b/LibGit2Sharp.Tests/AttributesFixture.cs index 8fea41f33..b9c2ba2f5 100644 --- a/LibGit2Sharp.Tests/AttributesFixture.cs +++ b/LibGit2Sharp.Tests/AttributesFixture.cs @@ -40,7 +40,7 @@ private static void AssertNormalization(Repository repo, string filename, bool s var blob = repo.Lookup(entry.Id); Assert.NotNull(blob); - Assert.Equal(!shouldHaveBeenNormalized, blob.ContentAsUtf8().Contains("\r")); + Assert.Equal(!shouldHaveBeenNormalized, blob.ContentAsText().Contains("\r")); } private static void CreateAttributesFile(Repository repo) diff --git a/LibGit2Sharp.Tests/BlobFixture.cs b/LibGit2Sharp.Tests/BlobFixture.cs index 2e09fad89..a7521df2a 100644 --- a/LibGit2Sharp.Tests/BlobFixture.cs +++ b/LibGit2Sharp.Tests/BlobFixture.cs @@ -1,24 +1,60 @@ using System.IO; +using System.Linq; using System.Text; using LibGit2Sharp.Tests.TestHelpers; using Xunit; +using Xunit.Extensions; namespace LibGit2Sharp.Tests { public class BlobFixture : BaseFixture { [Fact] - public void CanGetBlobAsUtf8() + public void CanGetBlobAsText() { using (var repo = new Repository(BareTestRepoPath)) { var blob = repo.Lookup("a8233120f6ad708f843d861ce2b7228ec4e3dec6"); - string text = blob.ContentAsUtf8(); + var text = blob.ContentAsText(); + Assert.Equal("hey there\n", text); } } + [Theory] + [InlineData("ascii", 4, "31 32 33 34")] + [InlineData("utf-7", 4, "31 32 33 34")] + // [InlineData("utf-8", 7, "EF BB BF 31 32 33 34")] // Fails due to BOM + // [InlineData("utf-16", 10, "FF FE 31 00 32 00 33 00 34 00")] // Fails due to BOM + // [InlineData("unicodeFFFE", 10, "FE FF 00 31 00 32 00 33 00 34")] // Fails due to BOM + // [InlineData("utf-32", 20, "FF FE 00 00 31 00 00 00 32 00 00 00 33 00 00 00 34 00 00 00")] // Fails due to BOM + public void CanGetBlobAsTextWithVariousEncodings(string encodingName, int expectedContentBytes, string expectedUtf7Chars) + { + var path = CloneStandardTestRepo(); + using (var repo = new Repository(path)) + { + var bomFile = "bom.txt"; + var content = "1234"; + var encoding = Encoding.GetEncoding(encodingName); + + var bomPath = Touch(repo.Info.WorkingDirectory, bomFile, content, encoding); + Assert.Equal(expectedContentBytes, File.ReadAllBytes(bomPath).Length); + + repo.Index.Stage(bomFile); + var commit = repo.Commit("bom", Constants.Signature, Constants.Signature); + + var blob = (Blob)commit.Tree[bomFile].Target; + Assert.Equal(expectedContentBytes, blob.Content.Length); + + var text = blob.ContentAsText(encoding); + Assert.Equal(content, text); + + var utf7Chars = blob.ContentAsText(Encoding.UTF7).Select(c => ((int)c).ToString("X2")).ToArray(); + Assert.Equal(expectedUtf7Chars, string.Join(" ", utf7Chars)); + } + } + [Fact] public void CanGetBlobSize() { diff --git a/LibGit2Sharp.Tests/CommitFixture.cs b/LibGit2Sharp.Tests/CommitFixture.cs index b6577d453..0ad0e1fbd 100644 --- a/LibGit2Sharp.Tests/CommitFixture.cs +++ b/LibGit2Sharp.Tests/CommitFixture.cs @@ -480,7 +480,7 @@ public void CanDirectlyAccessABlobOfTheCommit() var blob = commit["1/branch_file.txt"].Target as Blob; Assert.NotNull(blob); - Assert.Equal("hi\n", blob.ContentAsUtf8()); + Assert.Equal("hi\n", blob.ContentAsText()); } } @@ -694,7 +694,7 @@ public void CanCommitALittleBit() private static void AssertBlobContent(TreeEntry entry, string expectedContent) { Assert.Equal(TreeEntryTargetType.Blob, entry.TargetType); - Assert.Equal(expectedContent, ((Blob)(entry.Target)).ContentAsUtf8()); + Assert.Equal(expectedContent, ((Blob)(entry.Target)).ContentAsText()); } private static void AddCommitToRepo(string path) diff --git a/LibGit2Sharp.Tests/ObjectDatabaseFixture.cs b/LibGit2Sharp.Tests/ObjectDatabaseFixture.cs index 4804a3bb2..90673b95f 100644 --- a/LibGit2Sharp.Tests/ObjectDatabaseFixture.cs +++ b/LibGit2Sharp.Tests/ObjectDatabaseFixture.cs @@ -68,7 +68,7 @@ public void CanCreateABlobIntoTheDatabaseOfABareRepository() Assert.NotNull(blob); Assert.Equal("dc53d4c6b8684c21b0b57db29da4a2afea011565", blob.Sha); - Assert.Equal("I'm a new file\n", blob.ContentAsUtf8()); + Assert.Equal("I'm a new file\n", blob.ContentAsText()); var fetchedBlob = repo.Lookup(blob.Id); Assert.Equal(blob, fetchedBlob); diff --git a/LibGit2Sharp/BlobExtensions.cs b/LibGit2Sharp/BlobExtensions.cs index 110cdadb8..67b8d1cbc 100644 --- a/LibGit2Sharp/BlobExtensions.cs +++ b/LibGit2Sharp/BlobExtensions.cs @@ -1,4 +1,6 @@ -using System.Text; +using System; +using System.IO; +using System.Text; using LibGit2Sharp.Core; namespace LibGit2Sharp @@ -8,16 +10,32 @@ namespace LibGit2Sharp /// public static class BlobExtensions { + /// + /// Gets the blob content decoded with the specified encoding, + /// or UTF8 if encoding is not specified. + /// + /// The blob for which the content will be returned. + /// The encoding of the text. (default: UTF8) + /// Blob content as text. + public static string ContentAsText(this Blob blob, Encoding encoding = null) + { + Ensure.ArgumentNotNull(blob, "blob"); + encoding = encoding ?? Encoding.UTF8; + + return encoding.GetString(blob.Content); + } + /// /// Gets the blob content decoded as UTF-8. /// /// The blob for which the content will be returned. /// Blob content as UTF-8 + [Obsolete("This method will be removed in the next release. Please use ContentAsText()")] public static string ContentAsUtf8(this Blob blob) { Ensure.ArgumentNotNull(blob, "blob"); - return Encoding.UTF8.GetString(blob.Content); + return blob.ContentAsText(Encoding.UTF8); } /// @@ -25,11 +43,12 @@ public static string ContentAsUtf8(this Blob blob) /// /// The blob for which the content will be returned. /// Blob content as unicode. + [Obsolete("This method will be removed in the next release. Please use ContentAsText()")] public static string ContentAsUnicode(this Blob blob) { Ensure.ArgumentNotNull(blob, "blob"); - return Encoding.Unicode.GetString(blob.Content); + return blob.ContentAsText(Encoding.Unicode); } } } From e02eef9408d5fa91b838fc8c99d0842dab7d5490 Mon Sep 17 00:00:00 2001 From: Keith Dahlby Date: Sun, 4 Aug 2013 21:52:47 -0500 Subject: [PATCH 3/3] Support BOM detection in Blob.GetContentAsText() Fixes #475 --- LibGit2Sharp.Tests/BlobFixture.cs | 11 +++++++---- LibGit2Sharp/BlobExtensions.cs | 11 +++++++---- 2 files changed, 14 insertions(+), 8 deletions(-) diff --git a/LibGit2Sharp.Tests/BlobFixture.cs b/LibGit2Sharp.Tests/BlobFixture.cs index a7521df2a..6d97a588d 100644 --- a/LibGit2Sharp.Tests/BlobFixture.cs +++ b/LibGit2Sharp.Tests/BlobFixture.cs @@ -25,10 +25,10 @@ public void CanGetBlobAsText() [Theory] [InlineData("ascii", 4, "31 32 33 34")] [InlineData("utf-7", 4, "31 32 33 34")] - // [InlineData("utf-8", 7, "EF BB BF 31 32 33 34")] // Fails due to BOM - // [InlineData("utf-16", 10, "FF FE 31 00 32 00 33 00 34 00")] // Fails due to BOM - // [InlineData("unicodeFFFE", 10, "FE FF 00 31 00 32 00 33 00 34")] // Fails due to BOM - // [InlineData("utf-32", 20, "FF FE 00 00 31 00 00 00 32 00 00 00 33 00 00 00 34 00 00 00")] // Fails due to BOM + [InlineData("utf-8", 7, "EF BB BF 31 32 33 34")] + [InlineData("utf-16", 10, "FF FE 31 00 32 00 33 00 34 00")] + [InlineData("unicodeFFFE", 10, "FE FF 00 31 00 32 00 33 00 34")] + [InlineData("utf-32", 20, "FF FE 00 00 31 00 00 00 32 00 00 00 33 00 00 00 34 00 00 00")] public void CanGetBlobAsTextWithVariousEncodings(string encodingName, int expectedContentBytes, string expectedUtf7Chars) { var path = CloneStandardTestRepo(); @@ -47,6 +47,9 @@ public void CanGetBlobAsTextWithVariousEncodings(string encodingName, int expect var blob = (Blob)commit.Tree[bomFile].Target; Assert.Equal(expectedContentBytes, blob.Content.Length); + var textDetected = blob.ContentAsText(); + Assert.Equal(content, textDetected); + var text = blob.ContentAsText(encoding); Assert.Equal(content, text); diff --git a/LibGit2Sharp/BlobExtensions.cs b/LibGit2Sharp/BlobExtensions.cs index 67b8d1cbc..4d673ff0a 100644 --- a/LibGit2Sharp/BlobExtensions.cs +++ b/LibGit2Sharp/BlobExtensions.cs @@ -12,17 +12,20 @@ public static class BlobExtensions { /// /// Gets the blob content decoded with the specified encoding, - /// or UTF8 if encoding is not specified. + /// or according to byte order marks, with UTF8 as fallback, + /// if is null. /// /// The blob for which the content will be returned. - /// The encoding of the text. (default: UTF8) + /// The encoding of the text. (default: detected or UTF8) /// Blob content as text. public static string ContentAsText(this Blob blob, Encoding encoding = null) { Ensure.ArgumentNotNull(blob, "blob"); - encoding = encoding ?? Encoding.UTF8; - return encoding.GetString(blob.Content); + using (var reader = new StreamReader(blob.ContentStream, encoding ?? Encoding.UTF8, encoding == null)) + { + return reader.ReadToEnd(); + } } ///