fix error when response is not 200

Author: dongguaguaguagua

src/books_crawler.py

@@ -54,7 +54,7 @@ def from_user_get_books(db_path, data_count, user_id, round):
     init_books_table(db_path)
     insert_books(db_path, data)
 
-    if(start > data.get('total')):
+    if(start > data.get('total', start)):
         return False
     return True
 
@@ -84,7 +84,7 @@ def from_doulist_get_books(db_path, data_count, doulist_id, round):
     init_books_table(db_path)
     insert_books(db_path, data)
 
-    if(start > data.get('total')):
+    if(start > data.get('total', start)):
         return False
     return True
 
@@ -128,10 +128,10 @@ def get_recommendations(db_path, data_count, round):
     print(f'-------- round: {round}, start: {start} --------')
     print(f'Total book count: {total_book_after}')
     print(f'Replaced books count: {replaced_book_count}')
-    print(f'Response total:{data.get('total')}')
+    print(f'Response total:{data.get('total', start)}')
     print(f"{received_count} Data has been inserted into the SQLite database.")
 
-    if(start > data.get('total')):
+    if(start > data.get('total', start)):
         return False
     return True
 

src/movies_crawler.py

@@ -54,7 +54,7 @@ def from_user_get_movies(db_path, data_count, user_id, round):
     init_movies_table(db_path)
     insert_movies(db_path, data)
 
-    if(start > data.get('total')):
+    if(start > data.get('total', start)):
         return False
     return True
 
@@ -84,7 +84,7 @@ def from_doulist_get_movies(db_path, data_count, doulist_id, round):
     init_movies_table(db_path)
     insert_movies(db_path, data)
 
-    if(start > data.get('total')):
+    if(start > data.get('total', start)):
         return False
     return True
 
@@ -128,10 +128,10 @@ def get_recommendations(db_path, data_count, round):
     log_message(f'-------- round: {round}, start: {start} --------','movie_log.txt')
     log_message(f'Total movie count: {total_movie_after}','movie_log.txt')
     log_message(f'Replaced movies count: {replaced_movie_count}','movie_log.txt')
-    log_message(f'Response total:{data.get('total')}','movie_log.txt')
+    log_message(f'Response total:{data.get('total', start)}','movie_log.txt')
     log_message(f"{received_count} Data has been inserted into the SQLite database.",'movie_log.txt')
 
-    if(start > data.get('total')):
+    if(start > data.get('total', start)):
         return False
     return True
 

src/test.py

@@ -3,6 +3,7 @@
 from utilities import *
 from get_headers import *
 from insert_data import *
+from movies_crawler import *
 # cookies = {
 #     'bid': '4Xl28mUhvMg',
 # }
@@ -43,4 +44,5 @@
 with open('latest_movies_from_doulist.json','r') as f:
     data = json.load(f)
 insert_doulists('movies.db', data)
-
+start=10
+print(start > data.get('total', start))