merging db more general

Author: dongguaguaguagua

src/merge_db.py

@@ -1,43 +1,23 @@
 import sqlite3
+from init_tables import *
 
-# 连接到两个源数据库
-conn = sqlite3.connect('movies3.db')
-
-# 连接到目标数据库
+conn = sqlite3.connect('movies的副本.db')
 conn_merged = sqlite3.connect('merged.db')
 
-# 创建游标对象
+
 cur1 = conn.cursor()
 cur_merged = conn_merged.cursor()
 
-# 创建目标数据库的movie表结构
-cur_merged.execute('''
-CREATE TABLE IF NOT EXISTS movies (
-    id INTEGER PRIMARY KEY,
-    title TEXT,
-    rating REAL,
-    rating_count INTEGER,
-    pubdate TEXT,
-    year TEXT,
-    genres TEXT,
-    durations TEXT,
-    cover_url TEXT,
-    sharing_url TEXT,
-    countries TEXT,
-    url TEXT,
-    directors TEXT,
-    actors TEXT,
-    update_time TEXT
-)
-''')
+init_movies_table('merged.db')
 
 # 从第一个数据库中插入数据
 cur1.execute('SELECT * FROM movies')
 rows1 = cur1.fetchall()
 cur_merged.executemany('''
-INSERT INTO movies (id, title, rating, rating_count, pubdate, year, genres, durations, cover_url,
-                   sharing_url, countries, url, directors, actors, update_time)
-VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
+INSERT INTO movies (id, title, rating, rating_count, pubdate,
+                    year, genres, durations, cover_url, sharing_url,
+                    countries, url, directors, actors, is_visited, update_time)
+VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
 ON CONFLICT(id) DO UPDATE SET
     title=excluded.title,
     rating=excluded.rating,

src/movies_crawler.py

@@ -150,19 +150,19 @@ def get_recommendations(db_path, data_count, round):
 
         log_message(f"getting related movies and doulists for movie: {movie_title}",'movie_log.txt')
         get_related(db_path, movie_id)
-        doulists = get_all_unvisited(db_path, 'doulists', 'id')
-        for doulist_id in doulists:
-            log_message(f"\tworking on doulist: {doulist_id}",'movie_log.txt')
-            crawl_entire_doulist(db_path, data_count, doulist_id)
-            mark_visited(db_path, 'doulists', 'id', doulist_id)
+        # doulists = get_all_unvisited(db_path, 'doulists', 'id')
+        # for doulist_id in doulists:
+        #     log_message(f"\tworking on doulist: {doulist_id}",'movie_log.txt')
+        #     crawl_entire_doulist(db_path, data_count, doulist_id)
+        #     mark_visited(db_path, 'doulists', 'id', doulist_id)
 
         log_message(f"getting related user comments for movie: {movie_title}",'movie_log.txt')
         get_interests(db_path, movie_id)
-        users = get_all_unvisited(db_path, 'interests', 'user_id')
-        for user_id in users:
-            log_message(f"\tworking on user:{user_id}",'movie_log.txt')
-            crawl_entire_user(db_path, data_count, user_id)
-            mark_visited(db_path, 'interests', 'user_id', user_id)
+        # users = get_all_unvisited(db_path, 'interests', 'user_id')
+        # for user_id in users:
+        #     log_message(f"\tworking on user:{user_id}",'movie_log.txt')
+        #     crawl_entire_user(db_path, data_count, user_id)
+        #     mark_visited(db_path, 'interests', 'user_id', user_id)
 
         mark_visited(db_path, 'movies', 'id', movie_id)
 

src/test.py

@@ -46,3 +46,4 @@
 insert_doulists('movies.db', data)
 start=10
 print(start > data.get('total', start))
+