网络用nodejs抓取Google Play游戏
#node #webscraping #serpapi

将被刮擦

what

完整代码

如果您不需要解释,请看一下the full code example in the online IDE

const puppeteer = require("puppeteer-extra");
const StealthPlugin = require("puppeteer-extra-plugin-stealth");

puppeteer.use(StealthPlugin());

const searchParams = {
  hl: "en", // Parameter defines the language to use for the Google search
  gl: "us", // parameter defines the country to use for the Google search
  device: "phone", // parameter defines the search device. Options: phone, tablet, tv, chromebook
};

const URL = `https://play.google.com/store/games?hl=${searchParams.hl}&gl=${searchParams.gl}&device=${searchParams.device}`;

async function scrollPage(page, scrollContainer) {
  let lastHeight = await page.evaluate(`document.querySelector("${scrollContainer}").scrollHeight`);
  while (true) {
    await page.evaluate(`window.scrollTo(0, document.querySelector("${scrollContainer}").scrollHeight)`);
    await page.waitForTimeout(4000);
    let newHeight = await page.evaluate(`document.querySelector("${scrollContainer}").scrollHeight`);
    if (newHeight === lastHeight) {
      break;
    }
    lastHeight = newHeight;
  }
}

async function getGamesFromPage(page) {
  const games = await page.evaluate(() => {
    const mainPageInfo = Array.from(document.querySelectorAll("section .oVnAB")).reduce((result, block) => {
      const categoryTitle = block.textContent.trim();
      const apps = Array.from(block.parentElement.querySelectorAll(".ULeU3b")).map((app) => {
        const link = `https://play.google.com${app.querySelector(".Si6A0c")?.getAttribute("href")}`;
        const appId = link.slice(link.indexOf("?id=") + 4);
        return {
          title: app.querySelector(".sT93pb.DdYX5.OnEJge")?.textContent.trim(),
          appCategory: app.querySelector(".sT93pb.w2kbF:not(.ePXqnb)")?.textContent.trim(),
          link,
          rating: parseFloat(app.querySelector(".ubGTjb:last-child > div")?.getAttribute("aria-label").slice(6, 9)),
          iconThumbnail: app.querySelector(".j2FCNc img")?.getAttribute("srcset").slice(0, -3),
          appThumbnail: app.querySelector(".Vc0mnc img")?.getAttribute("src") || app.querySelector(".Shbxxd img")?.getAttribute("src"),
          video: app.querySelector(".aCy7Gf button")?.getAttribute("data-video-url") || "No video preview",
          appId,
        };
      });
      return {
        ...result,
        [categoryTitle]: apps,
      };
    }, {});

    return mainPageInfo;
  });
  return games;
}

async function getMainPageInfo() {
  const browser = await puppeteer.launch({
    headless: true, // if you want to see what the browser is doing, you need to change this option to "false"
    args: ["--no-sandbox", "--disable-setuid-sandbox"],
  });

  const page = await browser.newPage();

  await page.setDefaultNavigationTimeout(60000);
  await page.goto(URL);

  await page.waitForSelector(".oVnAB");

  await scrollPage(page, ".T4LgNb");

  const games = await getGamesFromPage(page);

  await browser.close();

  return games;
}

getMainPageInfo().then((result) => console.dir(result, { depth: null }));

准备

首先,我们需要创建一个node.js* project并添加koude0koude1koude2koude3以控制铬(或chrome或firefox,但现在我们仅在DevTools Protocol上使用铬在headless或无头模式中。

在使用我们的项目的目录中,打开命令行并输入npm init -y,然后是npm i puppeteer puppeteer-extra puppeteer-extra-plugin-stealth

*如果您没有安装node.js,则可以download it from nodejs.org并遵循安装documentation

ð注意:另外,您可以使用puppeteer无需任何扩展即可,但是我强烈建议将其与puppeteer-extra一起使用puppeteer-extra-plugin-stealth,以防止您使用无头铬或正在使用web driver的网站检测。您可以在Chrome headless tests website上检查它。下面的屏幕截图显示了差异。

stealth

Process

首先,我们需要滚动所有游戏列表,直到没有更多的列表加载,这是下面描述的困难部分。

下一步是在滚动完成后从HTML元素中提取数据。通过SelectorGadget Chrome extension,获得合适的CSS选择器的过程非常容易,该过程能够通过单击浏览器中的所需元素来获取CSS选择器。但是,它并不总是完美地工作,尤其是当JavaScript大量使用该网站时。

如果您想了解更多有关它们的信息,我们在Serpapi上有专门的Web Scraping with CSS Selectors博客文章。

下面的GIF说明了使用Selectorgadget选择结果的不同部分的方法。

how

代码说明

声明koude1puppeteer-extra Library和koude11控制Chromium浏览器,以防止网站检测到您正在使用puppeteer-extra-plugin-stealth库中使用web driver

const puppeteer = require("puppeteer-extra");
const StealthPlugin = require("puppeteer-extra-plugin-stealth");

接下来,我们“说” puppeteer使用StealthPlugin,编写必要的请求参数并搜索URL:

puppeteer.use(StealthPlugin());

const searchParams = {
  hl: "en", // Parameter defines the language to use for the Google search
  gl: "us", // parameter defines the country to use for the Google search
  device: "phone", // parameter defines the search device. Options: phone, tablet, tv, chromebook
};

const URL = `https://play.google.com/store/games?hl=${searchParams.hl}&gl=${searchParams.gl}&device=${searchParams.device}`;

接下来,我们编写一个函数以滚动页面以加载所有文章:

async function scrollPage(page, scrollContainer) {
  ...
}

在此功能中,首先,我们需要获得scrollContainer高度(使用koude16方法)。然后,我们使用while循环,其中我们向下滚动scrollContainer,等待2秒(使用koude19方法),然后获得一个新的scrollContainer高度。

接下来,我们检查newHeight是否等于lastHeight我们停止循环。否则,我们将newHeight值定义为lastHeight变量,然后重复重复直到页面不向下滚动到末尾:

let lastHeight = await page.evaluate(`document.querySelector("${scrollContainer}").scrollHeight`);
while (true) {
  await page.evaluate(`window.scrollTo(0, document.querySelector("${scrollContainer}").scrollHeight)`);
  await page.waitForTimeout(4000);
  let newHeight = await page.evaluate(`document.querySelector("${scrollContainer}").scrollHeight`);
  if (newHeight === lastHeight) {
    break;
  }
  lastHeight = newHeight;
}

接下来,我们编写一个函数以从页面获取游戏数据:

async function getGamesFromPage(page) {
  ...
}

在此功能中,我们从页面上下文中获取信息并将其保存在返回的对象中。接下来,我们需要使用"section .oVnAB"选择器(koude26方法)获取所有HTML元素。然后,我们使用koude27方法(允许用结果使对象)迭代使用koude28方法构建的数组:

const games = await page.evaluate(() => {
  const mainPageInfo = Array.from(document.querySelectorAll("section .oVnAB")).reduce((result, block) => {
      ...
    }, {});

    return mainPageInfo;
});
return games;

最后,我们需要获得categoryTitletitleappCategorylinkratingratingiconThumbnailiconThumbnailappThumbnailappId(我们可以使用koude38koude39koude39方法从link删除每个应用程序的koude26koude26)(koude26)(koude26)(koude26)21( ,koude42koude43koude44方法。

在每个ITARATION步骤中,我们返回上一个步骤结果(使用koude45),并添加带有categoryTitle常数名称的新类别:

const categoryTitle = block.textContent.trim();
const apps = Array.from(block.parentElement.querySelectorAll(".ULeU3b")).map((app) => {
  const link = `https://play.google.com${app.querySelector(".Si6A0c")?.getAttribute("href")}`;
  const appId = link.slice(link.indexOf("?id=") + 4);
  return {
    title: app.querySelector(".sT93pb.DdYX5.OnEJge")?.textContent.trim(),
    appCategory: app.querySelector(".sT93pb.w2kbF:not(.ePXqnb)")?.textContent.trim(),
    link,
    rating: parseFloat(app.querySelector(".ubGTjb:last-child > div")?.getAttribute("aria-label").slice(6, 9)),
    iconThumbnail: app.querySelector(".j2FCNc img")?.getAttribute("srcset").slice(0, -3),
    appThumbnail: app.querySelector(".Vc0mnc img")?.getAttribute("src") || app.querySelector(".Shbxxd img")?.getAttribute("src"),
    video: app.querySelector(".aCy7Gf button")?.getAttribute("data-video-url") || "No video preview",
    appId,
  };
});
return {
  ...result,
  [categoryTitle]: apps,
};

接下来,编写一个函数来控制浏览器并获取信息:

async function getJobsInfo() {
  ...
}

首先,在此功能中,我们需要使用带有当前optionspuppeteer.launch({options})方法来定义browser,例如headless: trueargs: ["--no-sandbox", "--disable-setuid-sandbox"]

这些选项意味着我们将headless模式和数组与arguments一起使用,我们用来允许在线IDE中启动浏览器流程。然后我们打开一个新的page

const browser = await puppeteer.launch({
  headless: true, // if you want to see what the browser is doing, you need to change this option to "false"
  args: ["--no-sandbox", "--disable-setuid-sandbox"],
});

const page = await browser.newPage();

接下来,我们更改默认值(30 sec)等待选择器的时间到60000毫秒(1分钟)与koude53方法缓慢连接,请使用koude55方法访问URL,并使用koude56方法,以等待选择器加载:< br>

await page.setDefaultNavigationTimeout(60000);
await page.goto(URL);
await page.waitForSelector(".oVnAB");

最后,我们等到页面滚动,从games常数中保存游戏数据,关闭浏览器,然后返回收到的数据:

await scrollPage(page, ".T4LgNb");

const games = await getGamesFromPage(page);

await browser.close();

return games;

现在我们可以启动我们的解析器:

$ node YOUR_FILE_NAME # YOUR_FILE_NAME is the name of your .js file

输出

{
   "Simulation games":[
      {
         "title":"Minecraft",
         "appCategory":"Arcade",
         "link":"https://play.google.com/store/apps/details?id=com.mojang.minecraftpe",
         "rating":4.6,
         "iconThumbnail":"https://play-lh.googleusercontent.com/VSwHQjcAttxsLE47RuS4PqpC4LT7lCoSjE7Hx5AW_yCxtDvcnsHHvm5CTuL5BPN-uRTP=s64-rw",
         "appThumbnail":"https://i.ytimg.com/vi/-0PjOvHeGiY/hqdefault.jpg",
         "video":"https://play.google.com/video/lava/web/player/yt:movie:-0PjOvHeGiY?autoplay=1&embed=play",
         "appId":"com.mojang.minecraftpe"
      },
      ... and other results
   ],
   "Stylized games":[
      {
         "title":"BASEBALL 9",
         "appCategory":"Sports",
         "link":"https://play.google.com/store/apps/details?id=us.kr.baseballnine",
         "rating":4.5,
         "iconThumbnail":"https://play-lh.googleusercontent.com/Y9GghCslNNk-nWccWvxgCDYvd7yLSwkw0hZ3NVTkSJqigY0ZozK3lv85jR02XOJSDPQ=s64-rw",
         "appThumbnail":"https://i.ytimg.com/vi/f8d8LpagbVg/hqdefault.jpg",
         "video":"https://play.google.com/video/lava/web/player/yt:movie:f8d8LpagbVg?autoplay=1&embed=play",
         "appId":"us.kr.baseballnine"
      },
      ... and other results
   ],
    ... and other categories
}

usuingaoqian 38 from serpapi

本节是为了显示DIY解决方案与我们的解决方案之间的比较。

最大的区别是您不需要从头开始创建解析器并维护它。

也有可能在Google的某个时候阻止请求,我们在后端处理它,因此无需弄清楚如何自己做或弄清楚要使用哪个验证码,代理提供商。

首先,我们需要安装koude58

npm i google-search-results-nodejs

这是full code example,如果您不需要说明:

const SerpApi = require("google-search-results-nodejs");
const search = new SerpApi.GoogleSearch(process.env.API_KEY); //your API key from serpapi.com

const params = {
  engine: "google_play", // search engine
  gl: "us", // parameter defines the country to use for the Google search
  hl: "en", // parameter defines the language to use for the Google search
  store: "games", // parameter defines the type of Google Play store
  store_device: "phone", // parameter defines the search device. Options: phone, tablet, tv, chromebook, watch, car
};

const getJson = () => {
  return new Promise((resolve) => {
    search.json(params, resolve);
  });
};

const getResults = async () => {
  const json = await getJson();
  const appsResults = json.organic_results.reduce((result, category) => {
    const { title: categoryTitle, items } = category;
    const apps = items.map((app) => {
      const { title, link, rating, category, video = "No video preview", thumbnail, product_id } = app;
      return {
        title,
        link,
        rating,
        category,
        video,
        thumbnail,
        appId: product_id,
      };
    });
    return {
      ...result,
      [categoryTitle]: apps,
    };
  }, {});
  return appsResults;
};

getResults().then((result) => console.dir(result, { depth: null }));

代码说明

首先,我们需要从koude58库中声明SerpApi,并使用SerpApi的API键定义新的search实例:

const SerpApi = require("google-search-results-nodejs");
const search = new SerpApi.GoogleSearch(API_KEY);

接下来,我们为提出请求的必要参数编写:

const params = {
  engine: "google_play", // search engine
  gl: "us", // parameter defines the country to use for the Google search
  hl: "en", // parameter defines the language to use for the Google search
  store: "apps", // parameter defines the type of Google Play store
  store_device: "phone", // parameter defines the search device. Options: phone, tablet, tv, chromebook, watch, car
};

接下来,我们从Serpapi库中包装搜索方法,以便进一步处理搜索结果:

const getJson = () => {
  return new Promise((resolve) => {
    search.json(params, resolve);
  });
};

最后,我们声明了从页面获取数据并返回的函数getResult

const getResults = async () => {
  ...
};

首先,在此功能中,我们获得了带有结果的json,然后我们需要在接收到的json中迭代organic_results数组。为此,我们使用koude27方法(它允许用结果使对象)。在每个ITARATION步骤中,我们返回上一个步骤结果(使用koude45),并添加带有categoryTitle常数名称的新类别:

  const json = await getJson();
  const appsResults = json.organic_results.reduce((result, category) => {
    ...
    return {
      ...result,
      [categoryTitle]: apps,
    };
  }, {});
  return appsResults;

接下来,我们破坏了category元素,将title重新定义为categoryTitle常数,然后itarate items阵列以从该类别中获取所有游戏。为此,我们需要破坏app元素,为video设置默认值“无视频预览”(因为并非所有游戏都有视频预览)并返回此常数:

const { title: categoryTitle, items } = category;
const apps = items.map((app) => {
  const { title, link, rating, category, video = "No video preview", thumbnail, product_id } = app;
  return {
    title,
    link,
    rating,
    category,
    video,
    thumbnail,
    appId: product_id,
  };
});

之后,我们运行getResults函数并使用koude76方法在控制台中打印所有接收的信息,该方法允许您使用带有必要参数的对象来更改默认输出选项:

getResults().then((result) => console.dir(result, { depth: null }));

输出

{
   "Casual games":[
      {
         "title":"UNO!™",
         "link":"https://play.google.com/store/apps/details?id=com.matteljv.uno",
         "rating":4.5,
         "category":"Card",
         "video":"https://play.google.com/video/lava/web/player/yt:movie:yoP0pK013pg?autoplay=1&embed=play",
         "thumbnail":"https://play-lh.googleusercontent.com/jX8uoop11adVRZTaBo6NERCBzUc5zuTBO_MUZCVcAPdWzcjXpXW8zs8-6fQoyFEbuyHw=s64-rw",
         "appId":"com.matteljv.uno"
      },
      ... and other results
   ],
   "Premium games":[
      {
         "title":"Grand Theft Auto: Vice City",
         "link":"https://play.google.com/store/apps/details?id=com.rockstargames.gtavc",
         "rating":4.2,
         "category":"Arcade",
         "video":"https://play.google.com/video/lava/web/player/yt:movie:Qsx6ZMfcPUs?autoplay=1&embed=play",
         "thumbnail":"https://play-lh.googleusercontent.com/nl1Y6bn06faVBuPEwWh5gInl_Zji3A5wTA4zscKDsJLXpcZ5C35F5zaGzEwCE0bKJ8Jr=s64-rw",
         "appId":"com.rockstargames.gtavc"
      },
      ... and other results
   ],
    ... and other categories
}


链接

如果您想在此博客文章中添加其他功能(例如,提取其他类别),或者您想查看Serpapi,write me a message的某些项目。


加入我们的Twitter | YouTube

添加Feature Requestð«或Bugð