feat: Add seq offload for 🔥FLUX.2 w/o parallel (#490)

* feat: support hybrid cache + tp for flux.2

* feat: enable seq offload for FLUX.2 w/ GPU=1

Author: DefTruth

examples/parallelism/run_flux2_tp.py

@@ -76,8 +76,12 @@
 
 if world_size < 4 and GiB() <= 48:
     assert not args.compile, "Compilation requires more GPU memory. Please disable it."
-    pipe.enable_model_cpu_offload(device=device)
-    print("Enabled model CPU offload.")
+    if world_size < 2:
+        pipe.enable_sequential_cpu_offload(device=device)
+        print("Enabled sequential CPU offload.")
+    else:
+        pipe.enable_model_cpu_offload(device=device)
+        print("Enabled model CPU offload.")
 else:
     pipe.to(device)