Better handling of long paths

By integrating the use of long path prefix (\?\\) and careful use of short
paths we can workaround most of the MSI API long path limitations. It's not
perfect as short paths that still exceed MAX_PATH will fail in most MSI APIs.
But accessing files placed in cabinets and copied around should now be fully
long path supported.

Fixes 3065 9115

Author: robmen

src/wix/WixToolset.Core.Burn/Bind/ProcessBundleSoftwareTagsCommand.cs

@@ -69,7 +69,7 @@ private static IEnumerable<SoftwareTag> CollectPackageTags(IntermediateSection s
                 {
                     var payload = payloadSymbolsById[msiPackage.PayloadRef];
 
-                    using (var db = new Database(payload.SourceFile.Path, OpenDatabase.ReadOnly))
+                    using (var db = Database.OpenAsReadOnly(payload.SourceFile.Path))
                     {
                         if (db.TableExists("SoftwareIdentificationTag"))
                         {

src/wix/WixToolset.Core.Burn/Bundles/ProcessMsiPackageCommand.cs

@@ -164,7 +164,7 @@ public WixBundleHarvestedMsiPackageSymbol HarvestPackage()
 
                 this.CheckIfWindowsInstallerFileTooLarge(this.PackagePayload.SourceLineNumbers, sourcePath, "MSI");
 
-                using (var db = new Database(sourcePath, OpenDatabase.ReadOnly))
+                using (var db = Database.OpenAsReadOnly(sourcePath))
                 {
                     // Read data out of the msi database...
                     using (var sumInfo = new SummaryInformation(db))

src/wix/WixToolset.Core.Burn/Bundles/ProcessMspPackageCommand.cs

@@ -98,7 +98,7 @@ private WixBundleHarvestedMspPackageSymbol HarvestPackage()
 
             try
             {
-                using (var db = new Database(sourcePath, OpenDatabase.ReadOnly | OpenDatabase.OpenPatchFile))
+                using (var db = Database.OpenAsReadOnly(sourcePath, asPatch: true))
                 {
                     // Read data out of the msp database...
                     using (var sumInfo = new SummaryInformation(db))

src/wix/WixToolset.Core.Native/LongPathUtil.cs

@@ -0,0 +1,78 @@
+// Copyright (c) .NET Foundation and contributors. All rights reserved. Licensed under the Microsoft Reciprocal License. See LICENSE.TXT file in the project root for full license information.
+
+namespace WixToolset.Core.Native
+{
+    using System.IO;
+    using System.Runtime.InteropServices;
+    using System.Text;
+
+    internal static class PathUtil
+    {
+        private const int MaxPath = 260;
+        private const string LongPathPrefix = @"\\?\";
+
+        public static bool CreateOrGetShortPath(string path, out string shortPath)
+        {
+            var fileCreated = false;
+
+            // The file must exist so we can get its short path.
+            if (!File.Exists(path))
+            {
+                using (File.Create(path))
+                {
+                }
+
+                fileCreated = true;
+            }
+
+            // Use the short path to avoid issues with long paths in the MSI API.
+            shortPath = GetShortPath(path);
+
+            return fileCreated;
+        }
+
+        public static string GetPrefixedLongPath(string path)
+        {
+            if (path.Length > MaxPath && !path.StartsWith(LongPathPrefix))
+            {
+                path = LongPathPrefix + path;
+            }
+
+            return path;
+        }
+
+        public static string GetShortPath(string longPath)
+        {
+            var path = GetPrefixedLongPath(longPath);
+
+            var buffer = new StringBuilder(MaxPath); // start with MAX_PATH.
+
+            var result = GetShortPathName(path, buffer, (uint)buffer.Capacity);
+
+            // If result > buffer.Capacity, reallocate and call again (even though we're usually using short names to avoid long path)
+            // so the short path result is still going to end up too long for APIs requiring a short path.
+            if (result > buffer.Capacity)
+            {
+                buffer = new StringBuilder((int)result);
+
+                result = GetShortPathName(path, buffer, (uint)buffer.Capacity);
+            }
+
+            // If we succeeded, return the short path without the prefix.
+            if (result > 0)
+            {
+                path = buffer.ToString();
+
+                if (path.StartsWith(LongPathPrefix))
+                {
+                    path = path.Substring(LongPathPrefix.Length);
+                }
+            }
+
+            return path;
+        }
+
+        [DllImport("kernel32.dll", CharSet = CharSet.Unicode, SetLastError = true)]
+        private static extern uint GetShortPathName(string lpszLongPath, StringBuilder lpszShortPath, uint cchBuffer);
+    }
+}

src/wix/WixToolset.Core.Native/Msi/Database.cs

@@ -18,7 +18,7 @@ public sealed class Database : MsiHandle
         /// </summary>
         /// <param name="path">Path to the database to be opened.</param>
         /// <param name="type">Persist mode to use when opening the database.</param>
-        public Database(string path, OpenDatabase type)
+        private Database(string path, OpenDatabase type)
         {
             var error = MsiInterop.MsiOpenDatabase(path, (IntPtr)type, out var handle);
             if (0 != error)
@@ -34,15 +34,90 @@ public Database(string path, OpenDatabase type)
         /// </summary>
         public static int MsiMaxStreamNameLength => MsiInterop.MsiMaxStreamNameLength;
 
+        /// <summary>
+        /// Creates a new <see cref="Database"/> with the specified path.
+        /// </summary>
+        /// <param name="path">Path of database to be created.</param>
+        /// <param name="asPatch">Indicates whether the database should be opened as a patch file.</param>
+        public static Database Create(string path, bool asPatch = false)
+        {
+            var fileCreated = false;
+            var mode = OpenDatabase.CreateDirect;
+
+            if (asPatch)
+            {
+                mode |= OpenDatabase.OpenPatchFile;
+            }
+
+            try
+            {
+                fileCreated = PathUtil.CreateOrGetShortPath(path, out var shortPath);
+
+                return new Database(shortPath, mode);
+            }
+            catch // cleanup on error if we created the short path file.
+            {
+                if (fileCreated)
+                {
+                    File.Delete(path);
+                }
+
+                throw;
+            }
+        }
+
+        /// <summary>
+        /// Opens an existing <see cref="Database"/> with the specified path.
+        /// </summary>
+        /// <param name="path">Path of database to open.</param>
+        /// <param name="transact">Indicates whether to open the database in transaction mode.</param>
+        /// <param name="asPatch">Indicates whether the database should be opened as a patch file.</param>
+        public static Database Open(string path, bool transact = false, bool asPatch = false)
+        {
+            var mode = transact ? OpenDatabase.Transact : OpenDatabase.Direct;
+
+            if (asPatch)
+            {
+                mode |= OpenDatabase.OpenPatchFile;
+            }
+
+            // Use the short path to avoid issues with long paths in the MSI API.
+            var shortPath = PathUtil.GetShortPath(path);
+
+            return new Database(shortPath, mode);
+        }
+
+        /// <summary>
+        /// Opens an existing <see cref="Database"/> with the specified path.
+        /// </summary>
+        /// <param name="path">Path of database to open.</param>
+        /// <param name="asPatch">Indicates whether the database should be opened as a patch file.</param>
+        public static Database OpenAsReadOnly(string path, bool asPatch = false)
+        {
+            var mode = OpenDatabase.ReadOnly;
+
+            if (asPatch)
+            {
+                mode |= OpenDatabase.OpenPatchFile;
+            }
+
+            // Use the short path to avoid issues with long paths in the MSI API.
+            var shortPath = PathUtil.GetShortPath(path);
+
+            return new Database(shortPath, mode);
+        }
+
         /// <summary>
         /// Apply a transform to the MSI.
         /// </summary>
         /// <param name="transformFile">Path to transform to apply.</param>
         public void ApplyTransform(string transformFile)
         {
+            var shortTransformFile = PathUtil.GetShortPath(transformFile);
+
             // get the curret validation bits
             var conditions = TransformErrorConditions.None;
-            using (var summaryInfo = new SummaryInformation(transformFile))
+            using (var summaryInfo = new SummaryInformation(shortTransformFile))
             {
                 try
                 {
@@ -65,7 +140,9 @@ public void ApplyTransform(string transformFile)
         /// <param name="errorConditions">Specifies the error conditions that are to be suppressed.</param>
         public void ApplyTransform(string transformFile, TransformErrorConditions errorConditions)
         {
-            var error = MsiInterop.MsiDatabaseApplyTransform(this.Handle, transformFile, errorConditions);
+            var shortTransformFile = PathUtil.GetShortPath(transformFile);
+
+            var error = MsiInterop.MsiDatabaseApplyTransform(this.Handle, shortTransformFile, errorConditions);
             if (0 != error)
             {
                 throw new MsiException(error);
@@ -119,7 +196,9 @@ public void Commit()
         /// shows which properties should be validated to verify that this transform can be applied to the database.</param>
         public void CreateTransformSummaryInfo(Database referenceDatabase, string transformFile, TransformErrorConditions errorConditions, TransformValidations validations)
         {
-            var error = MsiInterop.MsiCreateTransformSummaryInfo(this.Handle, referenceDatabase.Handle, transformFile, errorConditions, validations);
+            var shortTransformFile = PathUtil.GetShortPath(transformFile);
+
+            var error = MsiInterop.MsiCreateTransformSummaryInfo(this.Handle, referenceDatabase.Handle, shortTransformFile, errorConditions, validations);
             if (0 != error)
             {
                 throw new MsiException(error);
@@ -137,7 +216,9 @@ public void Import(string idtPath)
             var folderPath = Path.GetFullPath(Path.GetDirectoryName(idtPath));
             var fileName = Path.GetFileName(idtPath);
 
-            var error = MsiInterop.MsiDatabaseImport(this.Handle, folderPath, fileName);
+            var shortFolderPath = PathUtil.GetShortPath(folderPath);
+
+            var error = MsiInterop.MsiDatabaseImport(this.Handle, shortFolderPath, fileName);
             if (1627 == error) // ERROR_FUNCTION_FAILED
             {
                 throw new WixInvalidIdtException(idtPath);
@@ -161,7 +242,9 @@ public void Export(string tableName, string folderPath, string fileName)
                 folderPath = Environment.CurrentDirectory;
             }
 
-            var error = MsiInterop.MsiDatabaseExport(this.Handle, tableName, folderPath, fileName);
+            var shortFolderPath = PathUtil.GetShortPath(folderPath);
+
+            var error = MsiInterop.MsiDatabaseExport(this.Handle, tableName, shortFolderPath, fileName);
             if (0 != error)
             {
                 throw new MsiException(error);
@@ -177,13 +260,29 @@ public void Export(string tableName, string folderPath, string fileName)
         /// there are no differences between the two databases.</returns>
         public bool GenerateTransform(Database referenceDatabase, string transformFile)
         {
-            var error = MsiInterop.MsiDatabaseGenerateTransform(this.Handle, referenceDatabase.Handle, transformFile, 0, 0);
-            if (0 != error && 0xE8 != error) // ERROR_NO_DATA(0xE8) means no differences were found
+            var fileCreated = false;
+
+            try
             {
-                throw new MsiException(error);
+                fileCreated = PathUtil.CreateOrGetShortPath(transformFile, out var shortTransformFile);
+
+                var error = MsiInterop.MsiDatabaseGenerateTransform(this.Handle, referenceDatabase.Handle, shortTransformFile, 0, 0);
+                if (0 != error && 0xE8 != error) // ERROR_NO_DATA(0xE8) means no differences were found
+                {
+                    throw new MsiException(error);
+                }
+
+                return (0xE8 != error);
             }
+            catch // Cleanup on error
+            {
+                if (fileCreated)
+                {
+                    File.Delete(transformFile);
+                }
 
-            return (0xE8 != error);
+                throw;
+            }
         }
 
         /// <summary>

src/wix/WixToolset.Core.Native/Msi/Installer.cs

@@ -34,11 +34,13 @@ public static string ExtractPatchXml(string path)
             var buffer = new StringBuilder(65535);
             var size = buffer.Capacity;
 
-            var error = MsiInterop.MsiExtractPatchXMLData(path, 0, buffer, ref size);
+            var shortPath = PathUtil.GetShortPath(path);
+
+            var error = MsiInterop.MsiExtractPatchXMLData(shortPath, 0, buffer, ref size);
             if (234 == error)
             {
                 buffer.EnsureCapacity(++size);
-                error = MsiInterop.MsiExtractPatchXMLData(path, 0, buffer, ref size);
+                error = MsiInterop.MsiExtractPatchXMLData(shortPath, 0, buffer, ref size);
             }
 
             if (error != 0)
@@ -57,8 +59,12 @@ public static string ExtractPatchXml(string path)
         /// <param name="hash">Int array that receives the returned file hash information.</param>
         public static void GetFileHash(string filePath, int options, out int[] hash)
         {
-            var hashInterop = new MSIFILEHASHINFO();
-            hashInterop.FileHashInfoSize = 20;
+            var hashInterop = new MSIFILEHASHINFO
+            {
+                FileHashInfoSize = 20
+            };
+
+            filePath = PathUtil.GetPrefixedLongPath(filePath);
 
             var error = MsiInterop.MsiGetFileHash(filePath, Convert.ToUInt32(options), hashInterop);
             if (0 != error)
@@ -76,9 +82,9 @@ public static void GetFileHash(string filePath, int options, out int[] hash)
         }
 
         /// <summary>
-        /// Returns the version string and language string in the format that the installer 
-        /// expects to find them in the database.  If you just want version information, set 
-        /// lpLangBuf and pcchLangBuf to zero. If you just want language information, set 
+        /// Returns the version string and language string in the format that the installer
+        /// expects to find them in the database.  If you just want version information, set
+        /// lpLangBuf and pcchLangBuf to zero. If you just want language information, set
         /// lpVersionBuf and pcchVersionBuf to zero.
         /// </summary>
         /// <param name="filePath">Specifies the path to the file.</param>
@@ -91,6 +97,8 @@ public static void GetFileVersion(string filePath, out string version, out strin
             var versionBuffer = new StringBuilder(versionLength);
             var languageBuffer = new StringBuilder(languageLength);
 
+            filePath = PathUtil.GetPrefixedLongPath(filePath);
+
             var error = MsiInterop.MsiGetFileVersion(filePath, versionBuffer, ref versionLength, languageBuffer, ref languageLength);
             if (234 == error)
             {

src/wix/WixToolset.Core.Native/Msi/OpenDatabase.cs

@@ -1,4 +1,4 @@
-// Copyright (c) .NET Foundation and contributors. All rights reserved. Licensed under the Microsoft Reciprocal License. See LICENSE.TXT file in the project root for full license information.
+// Copyright (c) .NET Foundation and contributors. All rights reserved. Licensed under the Microsoft Reciprocal License. See LICENSE.TXT file in the project root for full license information.
 
 namespace WixToolset.Core.Native.Msi
 {

src/wix/WixToolset.Core.Native/Msi/SummaryInformation.cs

@@ -220,8 +220,10 @@ public SummaryInformation(string databaseFile)
                 throw new ArgumentNullException(nameof(databaseFile));
             }
 
+            var shortDatabaseFile = PathUtil.GetShortPath(databaseFile);
+
             var handle = IntPtr.Zero;
-            var error = MsiInterop.MsiGetSummaryInformation(IntPtr.Zero, databaseFile, 0, ref handle);
+            var error = MsiInterop.MsiGetSummaryInformation(IntPtr.Zero, shortDatabaseFile, 0, ref handle);
             if (0 != error)
             {
                 throw new MsiException(error);

src/wix/WixToolset.Core.Native/WindowsInstallerValidator.cs

@@ -93,7 +93,7 @@ private void RunValidations()
 
             try
             {
-                using (var database = new Database(this.DatabasePath, OpenDatabase.Direct))
+                using (var database = Database.Open(this.DatabasePath))
                 {
                     var propertyTableExists = database.TableExists("Property");
                     string productCode = null;
@@ -130,7 +130,7 @@ private void RunValidations()
 
                         try
                         {
-                            using (var cubeDatabase = new Database(findCubeFile.Path, OpenDatabase.ReadOnly))
+                            using (var cubeDatabase = Database.OpenAsReadOnly(findCubeFile.Path))
                             {
                                 try
                                 {

src/wix/WixToolset.Core.WindowsInstaller/Bind/BindTransformCommand.cs

@@ -418,8 +418,8 @@ public void Execute()
                 Directory.CreateDirectory(Path.GetDirectoryName(this.OutputPath));
 
                 // create the transform file
-                using (var targetDatabase = new Database(targetDatabaseFile, OpenDatabase.ReadOnly))
-                using (var updatedDatabase = new Database(updatedDatabaseFile, OpenDatabase.ReadOnly))
+                using (var targetDatabase = Database.OpenAsReadOnly(targetDatabaseFile))
+                using (var updatedDatabase = Database.OpenAsReadOnly(updatedDatabaseFile))
                 {
                     if (updatedDatabase.GenerateTransform(targetDatabase, this.OutputPath))
                     {