More models

.gitignore

@@ -5,3 +5,4 @@ screenshot*
 vimGPT/
 .env
 __pycache__/
+node_modules/

main.py

@@ -1,12 +1,9 @@
 import argparse
 import time
-
 from whisper_mic import WhisperMic
-
 import vision
 from vimbot import Vimbot
 
-
 def main(voice_mode):
     print("Initializing the Vimbot driver...")
     driver = Vimbot()
@@ -25,19 +22,18 @@ def main(voice_mode):
         print(f"Objective received: {objective}")
     else:
         objective = input("Please enter your objective: ")
+        model = input("Please enter a model to use (llava, gpt4v, cogvlm, claude): ")
 
     while True:
         time.sleep(1)
         print("Capturing the screen...")
         screenshot = driver.capture()
-
         print("Getting actions for the given objective...")
-        action = vision.get_actions(screenshot, objective)
+        action = vision.get_actions(screenshot, objective, model)
         print(f"JSON Response: {action}")
         if driver.perform_action(action):  # returns True if done
             break
 
-
 def main_entry():
     parser = argparse.ArgumentParser(description="Run the Vimbot with optional voice input.")
     parser.add_argument(
@@ -48,7 +44,6 @@ def main_entry():
     args = parser.parse_args()
     main(args.voice)
 
-
 if __name__ == "__main__":
     try:
         main_entry()

package.json

@@ -0,0 +1,16 @@
+{
+  "name": "vimgpt",
+  "version": "1.0.0",
+  "description": "Giving multimodal models an interface to play with.",
+  "main": "index.js",
+  "scripts": {
+    "test": "echo \"Error: no test specified\" && exit 1"
+  },
+  "keywords": [],
+  "author": "",
+  "license": "ISC",
+  "dependencies": {
+    "playwright": "^1.42.1",
+    "replicate": "^0.29.1"
+  }
+}