代码之家 › 专栏 › 技术社区 › SIM

不能在飞行中刮印链接

puppeteer promise web-scraping node.js

SIM · 技术社区 · 6 年前

我已经用英语写了一个剧本 node.js 刮 links 从一个网页的不同标题。当我执行下面的脚本时,我得到 undefined 打印在控制台而不是我在找你。我定义的选择器是准确的。

我不想把并返回结果;相反,我希望在飞行中打印它们。因为我对用 节点.js 结合 puppeteer

const puppeteer = require('puppeteer');
function run () {
    return new Promise(async (resolve, reject) => {
        try {
            const browser = await puppeteer.launch();
            const page = await browser.newPage();
            await page.goto("https://stackoverflow.com/questions/tagged/web-scraping");
            let url = await page.evaluate(() => {
                let items = document.querySelectorAll('a.question-hyperlink');
                items.forEach((item) => {
                    //would like to keep the following line intact 
                    console.log(item.getAttribute('href'));
                });
            })
            browser.close();
            return resolve(url);
        } catch (e) {
            return reject(e);
        }
    })
}
run().then(console.log).catch(console.error);

如果我考虑声明一个空数组,那么下面的脚本可以正常工作 results 结果 但我不想这样走。我想坚持我在上面尝试的方式,比如在打印结果的时候。

const puppeteer = require('puppeteer');
function run () {
    return new Promise(async (resolve, reject) => {
        try {
            const browser = await puppeteer.launch();
            const page = await browser.newPage();
            await page.goto("https://stackoverflow.com/questions/tagged/web-scraping");
            let urls = await page.evaluate(() => {
                let results = [];
                let items = document.querySelectorAll('a.question-hyperlink');
                items.forEach((item) => {
                    results.push({
                        url:  item.getAttribute('href'),
                    });
                });
                return results;
            })
            browser.close();
            return resolve(urls);
        } catch (e) {
            return reject(e);
        }
    })
}
run().then(console.log).catch(console.error);

再一次:我的问题是,我如何才能打印像这样的链接 console.log(item.getAttribute('href')); 在飞行中不存储在阵列中?

3 回复 | 直到 6 年前

Atishay Jain 6 年前

跑 console.log() 里面 evaluate()

page.on('console', obj => console.log(obj._text));

所以现在整个片段都是这样的

const puppeteer = require('puppeteer');
function run () {
    return new Promise(async (resolve, reject) => {
        try {
            const browser = await puppeteer.launch();
            const page = await browser.newPage();
            page.on('console', obj => console.log(obj._text));
            await page.goto("https://stackoverflow.com/questions/tagged/web-scraping");
            let url = await page.evaluate(() => {
                let items = document.querySelectorAll('a.question-hyperlink');
                items.forEach((item) => {
                    //would like to keep the following line intact 
                    console.log(item.getAttribute('href'));
                });
            })
            browser.close();
            return resolve(url);
        } catch (e) {
            return reject(e);
        }
    })
}
run().then(console.log).catch(console.error);

希望这有帮助

Jim Factor 6 年前

https://github.com/GoogleChrome/puppeteer/issues/628

await page.$$eval

async function getStackoverflowLinks(){
  return new Promise(async(resolve, reject)=>{
    console.log(`going to launch chromium via puppeteer`)
    const browser = await puppeteer.launch()
    console.log(`creating page/tab`)
    const page = await browser.newPage()
    await page.goto('https://stackoverflow.com/questions/tagged/web-scraping')
    console.log("fetched SO web-scraping, now parsing link href")

    let matches = await page.$$eval('a.question-hyperlink', hrefs=>hrefs.map((a)=>{
      return a.href
    })) // $$eval and map version, $$eval returns an array
    console.log("matches = ", matches.length)

    await browser.close()
    resolve(matches)
  })
}

getStackoverflowLinks()
.then(hrefs=>{
  console.log("hrefs: ", hrefs)
})

Md. Abu Taher 6 年前

注意事项,

async 函数将返回一个承诺。
new Promise

在这方面,您可以简单地使用 .console

page.on("console", msg => console.log(msg.text()));
await page.evaluate(async => {
  console.log("I will be printed on node console too")
})

高级用法已在上讨论过 this answer .