FIX: Unknown product dependencies due to missing pkg (#82)

Author: trinhngocthuyen

docs/res/unknown_deps_err.png

@@ -1,4 +1,53 @@
 [< Knowledge Base](README.md)
 
 # 🩺 Troubleshooting
-TBU
+
+### Unknown Product Dependencies
+
+<img src="res/unknown_deps_err.png" width="600px">
+
+If you encountered this issue, it's because there are some product dependencies that the tool failed to infer their packages. This case is likely to happen when having **local packages** that were not properly added to the project. Below is the instructions to resolve this issue (once only).
+
+After seeing this error, you should see `__unknown__/<Product>` (ex. `__unknown__/DebugKit`) in the lockfile (xccache.lock).
+```json
+"EX.xcodeproj": {
+  "packages": [
+  ],
+  "dependencies": {
+    "EX": [
+      "__unknown__/DebugKit" // <-- HERE
+    ]
+  }
+}
+```
+This mean, the tool cannot infer the package of `DebugKit`. So, what you need to do in this lockfile is:
+
+(1) Specify the package of that unknown product.
+```json
+"EX.xcodeproj": {
+  "packages": [
+    {
+      "path_from_root": "LocalPackages/core-utils" // <-- HERE
+    }
+  ]
+}
+```
+
+(2) Then, replace the `__unknown__` in the dependencies by the package slug (ex. changing `__unknown__/DebugKit` to `core-utils/DebugKit`).
+
+```json
+"EX.xcodeproj": {
+  "packages": [
+    {
+      "path_from_root": "LocalPackages/core-utils" // <-- HERE
+    }
+  ],
+  "dependencies": {
+    "EX": [
+      "core-utils/DebugKit" // <-- HERE
+    ]
+  }
+}
+```
+
+(3) After that, run the xccache workflow again.

docs/troubleshooting.md

@@ -2,43 +2,37 @@
   "EX.xcodeproj": {
     "packages": [
       {
-        "repositoryURL": "https://github.com/firebase/firebase-ios-sdk",
-        "requirement": {
-          "kind": "upToNextMajorVersion",
-          "minimumVersion": "11.4.0"
-        }
+        "path_from_root": "LocalPackages/core-utils"
       },
       {
-        "repositoryURL": "https://github.com/yeatse/KingfisherWebP",
+        "path_from_root": "LocalPackages/wizard"
+      },
+      {
+        "repositoryURL": "https://github.com/facebook/facebook-ios-sdk",
         "requirement": {
           "kind": "upToNextMajorVersion",
-          "minimumVersion": "1.6.0"
+          "minimumVersion": "9.0.0"
         }
       },
       {
-        "repositoryURL": "https://github.com/SwiftyBeaver/SwiftyBeaver",
+        "repositoryURL": "https://github.com/firebase/firebase-ios-sdk",
         "requirement": {
           "kind": "upToNextMajorVersion",
-          "minimumVersion": "2.1.1"
+          "minimumVersion": "11.4.0"
         }
       },
       {
-        "repositoryURL": "https://github.com/Moya/Moya",
+        "repositoryURL": "https://github.com/googlemaps/ios-maps-sdk",
         "requirement": {
           "kind": "upToNextMajorVersion",
-          "minimumVersion": "15.0.3"
+          "minimumVersion": "9.4.0"
         }
       },
       {
-        "relative_path": "LocalPackages/core-utils",
-        "path": null,
-        "path_from_root": "LocalPackages/core-utils"
-      },
-      {
-        "repositoryURL": "https://github.com/googlemaps/ios-maps-sdk",
+        "repositoryURL": "https://github.com/Moya/Moya",
         "requirement": {
           "kind": "upToNextMajorVersion",
-          "minimumVersion": "9.4.0"
+          "minimumVersion": "15.0.3"
         }
       },
       {
@@ -56,32 +50,34 @@
         }
       },
       {
-        "repositoryURL": "https://github.com/facebook/facebook-ios-sdk",
+        "repositoryURL": "https://github.com/SwiftyBeaver/SwiftyBeaver",
         "requirement": {
           "kind": "upToNextMajorVersion",
-          "minimumVersion": "9.0.0"
+          "minimumVersion": "2.1.1"
         }
       },
       {
-        "relative_path": "LocalPackages/wizard",
-        "path": null,
-        "path_from_root": "LocalPackages/wizard"
+        "repositoryURL": "https://github.com/yeatse/KingfisherWebP",
+        "requirement": {
+          "kind": "upToNextMajorVersion",
+          "minimumVersion": "1.6.0"
+        }
       }
     ],
     "dependencies": {
       "EX": [
-        "SwiftyBeaver/SwiftyBeaver",
-        "Moya/Moya",
-        "ios-maps-sdk/GoogleMaps",
+        "core-utils/DebugKit",
+        "core-utils/DisplayKit",
+        "core-utils/ResourceKit",
+        "core-utils/Swizzler",
         "facebook-ios-sdk/FacebookLogin",
         "firebase-ios-sdk/FirebaseCrashlytics",
+        "ios-maps-sdk/GoogleMaps",
+        "KingfisherWebP/KingfisherWebP",
+        "Moya/Moya",
         "SDWebImage/SDWebImage",
         "SnapKit/SnapKit-Dynamic",
-        "KingfisherWebP/KingfisherWebP",
-        "core-utils/DebugKit",
-        "core-utils/ResourceKit",
-        "core-utils/Swizzler",
-        "core-utils/DisplayKit",
+        "SwiftyBeaver/SwiftyBeaver",
         "wizard/Wizard"
       ],
       "EXTests": [

examples/xccache.lock

@@ -1,21 +1,24 @@
 class Hash
-  def deep_merge(other, uniq_block: nil, &block)
-    dup.deep_merge!(other, uniq_block: uniq_block, &block)
+  def deep_merge(other, uniq_block: nil, sort_block: nil, &block)
+    dup.deep_merge!(other, uniq_block: uniq_block, sort_block: sort_block, &block)
   end
 
-  def deep_merge!(other, uniq_block: nil, &block)
+  def deep_merge!(other, uniq_block: nil, sort_block: nil, &block)
     merge!(other) do |key, this_val, other_val|
       result = if this_val.is_a?(Hash) && other_val.is_a?(Hash)
-                 this_val.deep_merge(other_val, uniq_block: uniq_block, &block)
+                 this_val.deep_merge(other_val, uniq_block: uniq_block, sort_block: sort_block, &block)
                elsif this_val.is_a?(Array) && other_val.is_a?(Array)
                  this_val + other_val
                elsif block_given?
                  block.call(key, this_val, other_val)
                else
                  other_val
                end
-      next result if uniq_block.nil? || !result.is_a?(Array)
-      result.reverse.uniq(&uniq_block).reverse # prefer updates
+
+      # uniq by block, prefer updates
+      result = result.reverse.uniq(&uniq_block).reverse if uniq_block && result.is_a?(Array)
+      result = result.sort_by(&sort_block) if sort_block && result.is_a?(Array)
+      result
     end
   end
 end

lib/xccache/core/hash.rb

@@ -2,6 +2,33 @@
 
 module XCCache
   class Lockfile < JSONRepresentable
+    class Pkg < Hash
+      def self.from_h(h)
+        Pkg.new.merge(h)
+      end
+
+      def key
+        @key ||= ["repositoryURL", "path_from_root", "relative_path"].find { |x| key?(x) }
+      end
+
+      def id
+        self[key]
+      end
+
+      def local?
+        key != "repositoryURL"
+      end
+
+      def slug
+        @slug ||= File.basename(id, ".*")
+      end
+
+      def relative_path_from_dir(dir)
+        return id if key == "relative_path"
+        (Pathname.pwd / id).relative_path_from(dir) if key == "path_from_root"
+      end
+    end
+
     def hash_for_project(project)
       raw[project.display_name] ||= {}
     end
@@ -11,22 +38,31 @@ def product_dependencies_by_targets
     end
 
     def deep_merge!(hash)
-      keys = ["repositoryURL", "path_from_root", "relative_path"]
-      raw.deep_merge!(hash, uniq_block: proc do |h|
-        h.is_a?(Hash) && (k = keys.find { |k| h.key?(k) }) ? h[k] : h
-      end)
+      raw.deep_merge!(
+        hash,
+        uniq_block: proc { |h| h.is_a?(Hash) ? Pkg.from_h(h).id || h : h },
+        sort_block: proc { |x| x.to_s.downcase },
+      )
+      # After deep_merge, clear property cache
+      (instance_variables - %i[@path @raw]).each do |ivar|
+        remove_instance_variable(ivar)
+      end
     end
 
     def pkgs
-      @pkgs ||= raw.values.flat_map { |h| h["packages"] || [] }
+      @pkgs ||= raw.values.flat_map { |h| h["packages"] || [] }.map { |h| Pkg.from_h(h) }
     end
 
     def local_pkgs
-      @local_pkgs ||= pkgs.select { |h| h.key?("relative_path") || h.key?("path") }
+      @local_pkgs ||= pkgs.select(&:local?).uniq
     end
 
     def local_pkg_slugs
-      @local_pkg_slugs ||= local_pkgs.map { |h| File.basename(h["relative_path"] || h["path"]) }.uniq
+      @local_pkg_slugs ||= local_pkgs.map(&:slug).uniq
+    end
+
+    def known_product_dependencies
+      raw.empty? ? [] : product_dependencies.reject { |d| File.dirname(d) == "__unknown__" }
     end
 
     def product_dependencies
@@ -36,5 +72,20 @@ def product_dependencies
     def targets_data
       @targets_data ||= product_dependencies_by_targets.transform_keys { |k| "#{k}.xccache" }
     end
+
+    def verify!
+      known_slugs = pkgs.map(&:slug)
+      unknown = product_dependencies.reject { |d| known_slugs.include?(File.dirname(d)) }
+      return if unknown.empty?
+
+      UI.error! <<~DESC
+        Unknown product dependencies at #{path}:
+
+        #{unknown.sort.map { |d| "  • #{d}" }.join("\n")}
+
+        Refer to this doc for how to resolve this issue:
+          https://github.com/trinhngocthuyen/xccache/blob/main/docs/troubleshooting.md#unknown-product-dependencies
+      DESC
+    end
   end
 end

lib/xccache/core/lockfile.rb

@@ -34,10 +34,14 @@ def perform_install
 
     def sync_lockfile
       UI.info("Syncing lockfile")
+      known_dependencies = lockfile.known_product_dependencies
       update_projects do |project|
-        lockfile.deep_merge!(project.display_name => lockfile_hash_for_project(project))
+        lockfile.deep_merge!(
+          project.display_name => lockfile_hash_for_project(project, known_dependencies)
+        )
       end
       lockfile.save
+      lockfile.verify!
     end
 
     def lockfile
@@ -62,10 +66,15 @@ def update_projects
 
     private
 
-    def lockfile_hash_for_project(project)
+    def lockfile_hash_for_project(project, known_dependencies)
       deps_by_targets = project.targets.to_h do |target|
-        deps = target.non_xccache_pkg_product_dependencies.select(&:pkg).map { |d| "#{d.pkg.slug}/#{d.product_name}" }
-        [target.name, deps]
+        deps = target.non_xccache_pkg_product_dependencies.map do |dep|
+          next dep.full_name unless dep.pkg.nil?
+          known = known_dependencies.find { |x| File.basename(x) == dep.product_name }
+          UI.warn("-> Assuming #{known} for #{dep.full_name}".dark) if known
+          known || dep.full_name
+        end
+        [target.name, deps.sort]
       end
       {
         "packages" => project.non_xccache_pkgs.map(&:to_h),
@@ -75,50 +84,6 @@ def lockfile_hash_for_project(project)
 
     def verify_projects!
       raise "No projects detected. Are you running on the correct project directory?" if projects.empty?
-      projects.each { |project| verify_product_dependencies!(project) }
-    end
-
-    def verify_product_dependencies!(project)
-      invalid = project.targets.flat_map(&:pkg_product_dependencies).reject(&:pkg)
-      return if invalid.empty?
-
-      # Remove invalid product dependencies
-      project.targets.each do |target|
-        target.remove_pkg_product_dependencies { |d| d.pkg.nil? }
-      end
-      project.save
-
-      items_desc = invalid.map { |x| "• #{x.to_hash}" }.join("\n")
-      pkgs_desc = project.pkgs.map { |x| "• #{x.display_name}" }.join("\n")
-      UI.error! <<~DESC
-        Invalid product dependency:
-
-        #{items_desc}
-
-        REASON:
-          A product dependency must have a valid reference to its package.
-          In this case, the package reference might be missing, or having a mismatched UDID.
-            -----------------------------------------------------
-            78A6451F74BE75DC0D90CDBD /* PRODUCT-X */ = {
-              isa = XCSwiftPackageProductDependency;
-              <---- 👈 HERE: Missing `package`
-              productName = PRODUCT-X;
-            };
-            -----------------------------------------------------
-            78A6451F74BE75DC0D90CDBD /* PRODUCT-Y */ = {
-              isa = XCSwiftPackageProductDependency;
-              package = <MISMATCHED_UDID> /* XCRemoteSwiftPackageReference "PACKAGE-Y" */; <---- 👈 HERE: Mismatched UDID
-              productName = PRODUCT-Y;
-            };
-            -----------------------------------------------------
-        🚩 Are you sure the packages were added to the project? Found packages:
-        #{pkgs_desc}
-
-        ACTION:
-          - Ensure packages were added to the project
-          - The tool already removed those invalid products from the target.
-            Please help RE-ADD THOSE PRODUCTS to the target. Then re-run the command again.
-      DESC
     end
 
     def add_xccache_refs_to_project(project)

lib/xccache/installer.rb

@@ -19,7 +19,7 @@ def replace_binaries_for_project(project)
         project.add_xccache_pkg unless project.has_xccache_pkg?
         project.targets.each do |target|
           target.add_xccache_product_dependency unless target.has_xccache_product_dependency?
-          target.remove_pkg_product_dependencies { |d| !d.pkg.xccache_pkg? }
+          target.remove_pkg_product_dependencies { |d| d.pkg.nil? || !d.pkg.xccache_pkg? }
         end
         project.remove_pkgs(&:non_xccache_pkg?) unless config.keep_pkgs_in_project?
       end

lib/xccache/installer/use.rb

@@ -50,9 +50,7 @@ def ascii_plist_annotation
 
         def to_h
           {
-            "relative_path" => relative_path,
-            "path" => path,
-            "path_from_root" => absolute_path.relative_path_from(Pathname(".").expand_path).to_s,
+            "path_from_root" => absolute_path.relative_path_from(Pathname.pwd).to_s,
           }
         end