assert,util: fix deep comparison for sets and maps with mixed types

lib/internal/util/comparisons.js

@@ -669,8 +669,10 @@ function setObjectEquiv(array, a, b, mode, memo) {
         if (b.has(val1)) {
           continue;
         }
-      } else if (mode !== kLoose || b.has(val1)) {
+      } else if (b.has(val1)) {
         continue;
+      } else if (mode !== kLoose) {
+        return false;
       }
     }
 
@@ -816,11 +818,14 @@ function mapObjectEquiv(array, a, b, mode, memo) {
   const extraChecks = mode === kLoose || array.length !== a.size;
 
   for (const { 0: key1, 1: item1 } of a) {
-    if (extraChecks &&
-        (typeof key1 !== 'object' || key1 === null) &&
-        (mode !== kLoose ||
-          (b.has(key1) && innerDeepEqual(item1, b.get(key1), mode, memo)))) { // Mixed mode
-      continue;
+    if (extraChecks && (typeof key1 !== 'object' || key1 === null)) {
+      if (b.has(key1)) {
+        if (mode !== kLoose || innerDeepEqual(item1, b.get(key1), mode, memo)) {
+          continue;
+        }
+      } else if (mode !== kLoose) {
+        return false;
+      }
     }
 
     let innerStart = start;

test/parallel/test-assert-deep.js

@@ -634,6 +634,21 @@ test('Handle sparse arrays', () => {
   assertNotDeepOrStrict(a, b, AssertionError, { partial: 'pass' });
 });
 
+test('Handle sets and maps with mixed keys', () => {
+  // https://github.com/nodejs/node/issues/61386
+  const aSet = new Set([0, new Set([1, 2, 3]), new Set([4, 5, 6])]);
+  const bSet = new Set([
+    0,
+    new Set([1, new Set([2, 3]), new Set([20, 30])]),
+    new Set([4, new Set([5, 6]), new Set([50, 60])]),
+  ]);
+  assertNotDeepOrStrict(aSet, bSet);
+
+  const aMap = new Map([[0, 'zero'], [1, 'one'], [new Set([1, 2, 3]), 'A']]);
+  const bMap = new Map([[0, 'zero'], [new Set([1, 2, 3]), 'A'], [new Set([9]), 'B']]);
+  assertNotDeepOrStrict(aMap, bMap);
+});
+
 test('Handle different error messages', () => {
   const err1 = new Error('foo1');
   assertNotDeepOrStrict(err1, new Error('foo2'), assert.AssertionError);