Cloudflare captcha solving (#66)

* install puppeteer-captcha-plugin

* added cloudflare plugin and created action

* documentation

* export

* route

* add router

* version update and variables names

* readme

* yarn.lock

* Merged 2 endpoints into one. Deprecated recaptcha_solver endpoint

* README, version update and move closeOnEmpty stuff

* delete debug logs

* CloudflareCaptchaSolverPlugin

Author: MatthewZMSU

README.md

@@ -149,6 +149,29 @@ Example request body:
 }
 ```
 
+### **/captcha_solver**
+
+This POST method allows to solve captchas on the page.
+Before you use it you have to enable the corresponding puppeteer-extra plugin and provide `TOKEN_2CAPTCHA` environment variable.
+Currently available captcha types:
+* Recaptcha
+* Cloudflare
+
+Example request body:
+```json5
+{
+    "solveCloudflareCaptcha": true,
+    "solveRecaptcha": true,
+    "closeOnEmpty": false,
+    "waitOptions": {  // selector, xpath or timeout, same as in the goto method
+        "timeout": 5000, //default timeout is 1000ms
+    },
+    "navigationOptions": { // use if click triggers navigation to other page; same as in goXXX methods
+        "waitUntil": "domcontentloaded",
+    }
+}
+```
+
 ### **/compose**
 
 This POST method allows to combine several puppeteer actions into one.

actions/captcha_solver/cloudflare_captcha_solver.js

@@ -0,0 +1,28 @@
+/**
+ * The function solves cloudflare captcha on the page.
+ * If there is no cloudflare captcha on the page nothing will happen.
+ * If there is a cloudflare captcha the function solves it and then inserts the special code
+ * into the page automatically.
+ *
+ * Returns useful information about solving cloudflare captcha.
+ * For more information about return value visit
+ * https://www.npmjs.com/package/puppeteer-captcha-plugin
+ */
+exports.cloudflareCaptchaSolver = async function cloudflareCaptchaSolver(page, request) {
+    let cloudflareCaptchaData;
+
+    console.log("cloudflareCaptchaSolver");
+    if (request.body.solveCloudflareCaptcha) {
+        cloudflareCaptchaData = await page.solveCloudflareCaptcha();
+    } else {
+        cloudflareCaptchaData = await page.findCloudflareCaptcha();
+    }
+
+    if (request.body.navigationOptions) {
+        await page.waitForNavigation(request.body.navigationOptions);
+    }
+
+    return {
+        cloudflareCaptchaData: cloudflareCaptchaData,
+    }
+}

actions/captcha_solver/index.js

@@ -0,0 +1,40 @@
+const {recaptchaSolver} = require("./recaptcha_solver");
+const {cloudflareCaptchaSolver} = require("./cloudflare_captcha_solver");
+const utils = require("../../helpers/utils");
+
+
+DEFAULT_TIMEOUT = 1000;
+
+/**
+ * The function solves captchas on the page.
+ * Currently available captcha types: Recaptcha, Cloudflare.
+ * Only work with enabled PuppeteerExtra.
+ */
+exports.captchaSolver = async function captchaSolver(page, request) {
+    let responseData = {}
+    if ("solveCloudflareCaptcha" in request.body) {
+        Object.assign(responseData, await cloudflareCaptchaSolver(page, request));
+    } else {
+        console.log("No Solve CloudflareCaptcha");
+    }
+    if ("solveRecaptcha" in request.body) {
+        Object.assign(responseData, await recaptchaSolver(page, request));
+    }
+
+    if (request.body.navigationOptions) {
+        await page.waitForNavigation(request.body.navigationOptions);
+    }
+
+    const waitOptions = request.body.waitOptions || {timeout: DEFAULT_TIMEOUT};
+    const contents = await utils.getContents(page, waitOptions);
+
+    if (request.query.closePage ||
+        (request.body.closeOnEmpty && !responseData.recaptchaData?.captchas && !responseData.cloudflareCaptchaData?.data)) {
+        await page.close();
+    }
+
+    return {
+        ...contents,
+        ...responseData,
+    };
+}

actions/captcha_solver/recaptcha_solver.js

@@ -0,0 +1,27 @@
+/**
+ * The function solves recaptchas on the page.
+ * If there is no recaptcha on the page nothing will happen.
+ * If there is a recaptcha the function solves it and then inserts the special code
+ * into the page automatically.
+ *
+ * Returns useful information about recaptcha_solving.
+ * For more information about return value visit
+ * https://github.com/berstend/puppeteer-extra/tree/master/packages/puppeteer-extra-plugin-recaptcha#result-object
+ */
+exports.recaptchaSolver = async function recaptchaSolver(page, request) {
+    let recaptchaData;
+
+    if (request.body.solveRecaptcha) {
+        recaptchaData = await page.solveRecaptchas();
+    } else {
+        recaptchaData = await page.findRecaptchas();
+    }
+
+    if (request.body.navigationOptions) {
+        await page.waitForNavigation(request.body.navigationOptions);
+    }
+
+    return {
+        recaptchaData: recaptchaData,
+    }
+}

actions/compose.js

@@ -3,6 +3,7 @@ const exceptions = require("../helpers/exceptions");
 endpoint2action = {
     action: require("./action").action,
     click: require("./click").click,
+    captcha_solver: require("./captcha_solver").captchaSolver,
     fill_form: require("./fill_form").fillForm,
     back: require("./goback").goBack,
     forward: require("./goforward").goForward,

actions/recaptcha_solver.js

@@ -1,4 +1,4 @@
-const utils = require('../helpers/utils')
+const utils = require('../helpers/utils');
 const exceptions = require("../helpers/exceptions");
 
 const DEFAULT_TIMEOUT = 1000;  // 1 second
@@ -12,6 +12,8 @@ const DEFAULT_TIMEOUT = 1000;  // 1 second
  * Returns useful information about recaptcha_solving.
  * For more information about return value visit
  * https://github.com/berstend/puppeteer-extra/tree/master/packages/puppeteer-extra-plugin-recaptcha#result-object
+ *
+ * @deprecated Since version 0.4.0. Use captchaSolver router instead.
  */
 exports.recaptchaSolver = async function recaptchaSolver(page, request) {
     if (!("solve_recaptcha" in request.body)) {

app.js

@@ -4,13 +4,15 @@ const express = require('express');
 const puppeteer = require('puppeteer-extra')
 
 const RecaptchaPlugin = require('puppeteer-extra-plugin-recaptcha')
+const CaptchaPlugin = require('puppeteer-captcha-plugin').CloudflareCaptchaSolverPlugin
 const StealthPlugin = require('puppeteer-extra-plugin-stealth')
 const cookieParser = require('cookie-parser');
 const bodyParser = require('body-parser');
 const AsyncLock = require('async-lock');
 
 const indexRouter = require('./routes/index');
 const composeRouter = require('./routes/compose');
+const captchaRouter = require("./routes/captcha_solver");
 const healthCheckRouter = require('./routes/health_check');
 const gotoRouter = require('./routes/goto');
 const backRouter = require('./routes/goback');
@@ -56,7 +58,12 @@ async function setupBrowser() {
                         token: TOKEN_2CAPTCHA
                     }
                 })
-            )
+            );
+            puppeteer.use(
+                new CaptchaPlugin({
+                    token: TOKEN_2CAPTCHA,
+                })
+            );
         }
     } catch (error) {
         console.error('Failed to proceed 2captcha token:', error);
@@ -113,6 +120,7 @@ app.use(cookieParser());
 
 app.use('/', indexRouter);
 app.use('/compose', composeRouter);
+app.use('/captcha_solver', captchaRouter);
 app.use('/health_check', healthCheckRouter);
 app.use('/goto', gotoRouter);
 app.use('/back', backRouter);

package.json

@@ -1,6 +1,6 @@
 {
   "name": "scrapy-puppeteer-service",
-  "version": "0.3.13",
+  "version": "0.4.0",
   "private": true,
   "scripts": {
     "start": "node ./bin/www"
@@ -21,6 +21,7 @@
     "morgan": "~1.10.0",
     "npm-run-all": "^4.1.5",
     "puppeteer": "^23.4.0",
+    "puppeteer-captcha-plugin": "^0.1.0",
     "puppeteer-extra": "^3.3.6",
     "puppeteer-extra-plugin-recaptcha": "^3.6.8",
     "puppeteer-extra-plugin-stealth": "^2.11.2",

routes/captcha_solver.js

@@ -0,0 +1,24 @@
+const express = require('express');
+const router = express.Router();
+
+const {captchaSolver} = require('../actions/captcha_solver');
+const utils = require('../helpers/utils');
+
+router.post('/', async function (req, res, next) {
+    if (!process.env.TOKEN_2CAPTCHA) {
+        res.status(501);
+        res.send("TOKEN_2CAPTCHA is not provided!");
+        next();
+        return;
+    }
+
+    try {
+        let response = await utils.performAction(req, captchaSolver);
+        res.header('scrapy-puppeteer-service-context-id', response.contextId);
+        res.send(response)
+    } catch (e) {
+        next(e);
+    }
+});
+
+module.exports = router;

routes/recaptcha_solver.js

@@ -1,10 +1,12 @@
-const express = require('express')
-const router = express.Router()
+const express = require('express');
+const router = express.Router();
 
 const {recaptchaSolver} = require('../actions/recaptcha_solver');
-const utils = require('../helpers/utils')
-const exceptions = require('../helpers/exceptions');
+const utils = require('../helpers/utils');
 
+/**
+ * @deprecated Since version 0.4.0. Use captchaSolver router instead.
+ */
 router.post('/', async function (req, res, next) {
     if (!process.env.TOKEN_2CAPTCHA) {
         res.status(501);