[core] Add retry for reading snapshot when encountering MismatchedInputException (#6678)

Author: yuzelin

paimon-core/src/main/java/org/apache/paimon/utils/SnapshotManager.java

@@ -24,6 +24,7 @@
 import org.apache.paimon.table.Instant;
 
 import org.apache.paimon.shade.caffeine2.com.github.benmanes.caffeine.cache.Cache;
+import org.apache.paimon.shade.jackson2.com.fasterxml.jackson.databind.exc.MismatchedInputException;
 
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
@@ -770,12 +771,26 @@ public static Snapshot fromPath(FileIO fileIO, Path path) {
     }
 
     public static Snapshot tryFromPath(FileIO fileIO, Path path) throws FileNotFoundException {
-        try {
-            return Snapshot.fromJson(fileIO.readFileUtf8(path));
-        } catch (FileNotFoundException e) {
-            throw e;
-        } catch (IOException e) {
-            throw new RuntimeException("Fails to read snapshot from path " + path, e);
+        int retryNumber = 0;
+        MismatchedInputException exception = null;
+        while (retryNumber++ < 10) {
+            try {
+                return Snapshot.fromJson(fileIO.readFileUtf8(path));
+            } catch (MismatchedInputException e) {
+                // retry
+                exception = e;
+                try {
+                    Thread.sleep(1_000);
+                } catch (InterruptedException ie) {
+                    Thread.currentThread().interrupt();
+                    throw new RuntimeException(ie);
+                }
+            } catch (FileNotFoundException e) {
+                throw e;
+            } catch (IOException e) {
+                throw new RuntimeException("Fails to read snapshot from path " + path, e);
+            }
         }
+        throw new UncheckedIOException(exception);
     }
 }