Added memcache support for test status

Author: pmeenan

internal/process_test.py

@@ -1041,6 +1041,15 @@ def generate_har(self):
             else:
                 logging.debug("Not uploading HAR")
 
+            # mark successful tests as having been run in memcache
+            is_new_test = False
+            if 'memcache' in self.job:
+                is_new_test = self.job['memcache'].add(self.task['id'], 1)
+                if not is_new_test:
+                    logging.debug("Test %s already exists in memcache", self.task['id'])
+            
+            # TODO: change this to use memcache as the source of the test already having been processed
+            # if is_new_test and needs_upload and os.path.exists(har_file):
             if uploaded:
                 if self.job['success'] and 'bq_datastore' in self.job:
                     self.upload_bigquery(har, har_filename, self.job['bq_datastore'])

wptagent.py

@@ -44,6 +44,7 @@ def __init__(self, options, browsers):
         self.job = None
         self.beanstalk_job = None
         self.beanstalk = None
+        self.memcache = None
         self.task = None
         self.xvfb = None
         self.hostname = socket.gethostname()
@@ -240,6 +241,12 @@ def get_beanstalk_job(self):
             import zlib
             if self.beanstalk is None:
                 self.beanstalk = greenstalk.Client((self.options.beanstalk, 11300), encoding=None, watch='crawl', use='alive')
+            if self.memcache is None:
+                try:
+                    from pymemcache.client.base import Client
+                    self.memcache = Client(self.options.beanstalk, connect_timeout=10, timeout=10, no_delay=True)
+                except Exception:
+                    logging.exception("Error connecting to memcache")
             self.beanstalk_job = self.beanstalk.reserve(30)
             stats = self.beanstalk.stats_job(self.beanstalk_job)
             if stats and 'timeouts' in stats and stats['timeouts'] >= 3:
@@ -255,6 +262,8 @@ def get_beanstalk_job(self):
                 raw = self.beanstalk_job.body
                 test_json = json.loads(zlib.decompress(raw).decode())
                 self.job = self.wpt.process_job_json(test_json)
+                if self.memcache is not None:
+                    self.job['memcache'] = self.memcache
         except greenstalk.TimedOutError:
             pass
         except Exception: