如何限制节点中简化HTTP请求的内容长度响应?

const request = require('request');
const URL = 'http://de.releases.ubuntu.com/xenial/ubuntu-16.04.3-desktop-amd64.iso';
const MAX_SIZE = 10 * 1024 * 1024 // 10MB , maximum size to download
let total_bytes_read = 0;

1-如果来自服务器的响应是gzip压缩的,则应该 启用gzip选项。 https://github.com/request/request#examples 为了向后兼容,不支持响应压缩 违约要接受gzip压缩响应,请设置gzip选项 为true。

request
    .get({
        uri: URL,
        gzip: true
    })
    .on('error', function (error) {
        //TODO: error handling
        console.error('ERROR::', error);
    })
    .on('data', function (data) {
        // decompressed data 
        console.log('Decompressed  chunck Recived:' + data.length, ': Total downloaded:', total_bytes_read)
        total_bytes_read += data.length;
        if (total_bytes_read >= MAX_SIZE) {
            //TODO: handle exceeds max size event
            console.error("Request exceeds max size.");
            throw new Error('Request exceeds max size'); //stop
        }
    })
    .on('response', function (response) {
        response.on('data', function (chunk) {
            //compressed data
            console.log('Compressed  chunck Recived:' + chunk.length, ': Total downloaded:', total_bytes_read)
        });
    })
    .on('end', function () {
        console.log('Request completed! Total size downloaded:', total_bytes_read)
    });

注意: 如果服务器没有压缩响应,但您仍然使用gzip 选项/解压缩,然后解压缩区块(&D);原始块将是 相同的因此,您可以通过任何一种方式(从 解压/压缩块),但如果响应被压缩 您应该检查解压缩区块的大小限制

2-如果响应未压缩,则不需要gzip选项来 解压

request
    .get(URL)
    .on('error', function (error) {
        //TODO: error handling
        console.error('ERROR::', error);
    })
    .on('response', function (response) {
        response.on('data', function (chunk) {
            //compressed data
            console.log('Recived chunck:' + chunk.length, ': Total downloaded:', total_bytes_read)
            total_bytes_read += chunk.length;
            if (total_bytes_read >= MAX_SIZE) {
                //TODO: handle exceeds max size event
                console.error("Request as it exceds max size:")
                throw new Error('Request as it exceds max size');
            }
            console.log("...");
        });
    })
    .on('end', function () {
        console.log('Request completed! Total size downloaded:', total_bytes_read)
    });

您可以使用 data 在这种情况下的事件 request 包装也一样。我在下面进行了测试,效果很好

var request = require("request");

var size = 0;
const MAX_SIZE = 200;
request
    .get('http://google.com/')
    .on('data', function(buffer){
        // decompressed data as it is received

        size += buffer.length;

        if (size > MAX_SIZE) {
            console.log("Aborting this request as it exceeds max size")
            this.abort();
        }
        console.log("data coming");

    }).on('end', function() {
        console.log('ending request')
    })
    .on('response', function (response) {
        console.log(response.statusCode) // 200
        console.log(response.headers['content-type']) // 'image/png'
        response.on('data', function (data) {
            // compressed data as it is received
            console.log('received ' + data.length + ' bytes of compressed data')
            // you can size and abort here also if you want.
        })
    });

有两个地方可以进行大小检查,一个是获取压缩数据的地方,另一个是获取未压缩数据的地方(基于中给出的示例) https://www.npmjs.com/package/request )

正如@Jackthomson在第一条评论的回答中指出的那样,可以使用 .on(data) 如果您想要标题,可以从响应中获取它们,也可以检查 content-length 标题,而不是开始分块。

来自axios参考。

// maxContentLength 定义http响应的最大大小 允许的内容maxContentLength:2000,

axios就是这样处理的 最大内容长度

var responseBuffer = [];
        stream.on('data', function handleStreamData(chunk) {
          responseBuffer.push(chunk);

          // make sure the content length is not over the maxContentLength if specified
          if (config.maxContentLength > -1 && Buffer.concat(responseBuffer).length > config.maxContentLength) {
            reject(createError('maxContentLength size of ' + config.maxContentLength + ' exceeded',
              config, null, lastRequest));
          }
        });

部分的 request 相等的

var request = require("request");

const MAX_CONTENT_LENGTH = 10000000;

var receivedLength = 0;

var req = request.get('http://de.releases.ubuntu.com/xenial/ubuntu-16.04.3-desktop-amd64.iso')
    .on('response', (response) => {
        if (response.headers['content-length'] && response.headers['content-length'] > MAX_CONTENT_LENGTH) {
            console.log("max content-length exceeded")
            req.abort();
        }
    })
    .on('data', (str) => {
        receivedLength += str.length;
        if (receivedLength > MAX_CONTENT_LENGTH) {
            console.log("max content-length exceeded")
            req.abort();
        }
    })