代码之家  ›  专栏  ›  技术社区  ›  abhisek

亚马逊S3远程文件上传

  •  2
  • abhisek  · 技术社区  · 6 年前

    我正在尝试编写一个函数,它将:

    1. 以远程URL作为参数,
    2. 使用AXIOS获取文件
    3. 上传流到亚马逊S3
    4. 最后,返回上传的URL

    我找到了帮助 here on stackoverflow 。到目前为止,我有:

    /* 
     * Method to pipe the stream 
     */
    const uploadFromStream = (file_name, content_type) => {
      const pass = new stream.PassThrough();
    
      const obj_key = generateObjKey(file_name);
      const params = { Bucket: config.bucket, ACL: config.acl, Key: obj_key, ContentType: content_type, Body: pass };
    
      s3.upload(params, function(err, data) {
        if(!err){
            return data.Location;
        } else {
            console.log(err, data);
        }
      });
    
      return pass;
    }
    
    
    /*
     * Method to upload remote file to s3
     */
    const uploadRemoteFileToS3 = async (remoteAddr) => {
        axios({
            method: 'get',
            url: remoteAddr,
            responseType: 'stream'
        }).then( (response) => {
            if(response.status===200){
                const file_name = remoteAddr.substring(remoteAddr.lastIndexOf('/')+1);
                const content_type = response.headers['content-type'];
                response.data.pipe(uploadFromStream(file_name, content_type));
            }
        });
    }
    

    但是 uploadRemoteFileToS3 不返回任何内容(因为它是异步函数)。如何获取上载的URL?

    更新

    我进一步改进了代码,编写了一个类。以下是我现在拥有的:

    const config = require('../config.json');
    
    const stream = require('stream');
    const axios = require('axios');
    const AWS = require('aws-sdk');
    
    class S3RemoteUploader {
        constructor(remoteAddr){
            this.remoteAddr = remoteAddr;
            this.stream = stream;
            this.axios = axios;
            this.config = config;
            this.AWS = AWS;
            this.AWS.config.update({
                accessKeyId: this.config.api_key,
                secretAccessKey: this.config.api_secret
            });
            this.spacesEndpoint = new this.AWS.Endpoint(this.config.endpoint);
            this.s3 = new this.AWS.S3({endpoint: this.spacesEndpoint});
    
            this.file_name = this.remoteAddr.substring(this.remoteAddr.lastIndexOf('/')+1);
            this.obj_key = this.config.subfolder+'/'+this.file_name;
            this.content_type = 'application/octet-stream';
    
            this.uploadStream();
        }
    
        uploadStream(){
            const pass = new this.stream.PassThrough();
            this.promise = this.s3.upload({
                Bucket: this.config.bucket,
                Key: this.obj_key,
                ACL: this.config.acl,
                Body: pass,
                ContentType: this.content_type
            }).promise();
            return pass;
        }
    
        initiateAxiosCall() {
            axios({
                method: 'get',
                url: this.remoteAddr,
                responseType: 'stream'
            }).then( (response) => {
                if(response.status===200){
                    this.content_type = response.headers['content-type'];
                    response.data.pipe(this.uploadStream());
                }
            });
        }
    
        dispatch() {
            this.initiateAxiosCall();
        }
    
        async finish(){
            //console.log(this.promise); /* return Promise { Pending } */
            return this.promise.then( (r) => {
                console.log(r.Location);
                return r.Location;
            }).catch( (e)=>{
                console.log(e);
            });
        }
    
        run() {
            this.dispatch();
            this.finish();
        }
    }
    

    但是,当承诺得到解决时,仍然不知道如何抓住结果。到目前为止,我尝试过:

    testUpload = new S3RemoteUploader('https://avatars2.githubusercontent.com/u/41177');
    testUpload.run();
    //console.log(testUpload.promise); /* Returns Promise { Pending } */
    testUpload.promise.then(r => console.log); // does nothing
    

    但上述都不起作用。我觉得我错过了一些非常微妙的东西。有什么线索吗?

    2 回复  |  直到 6 年前
        1
  •  1
  •   Krishnadas PC    6 年前

    上传之后,您可以调用s3 sdk中的getsignedull函数来获取URL,在这里您还可以指定URL的到期时间。您需要传递该函数的键。现在旅行将在后面的示例中更新。

    生成一个简单的预签名URL,允许任何用户查看 在您拥有的桶中的私有对象的内容,可以使用 以下调用togetSignedURL():

     var s3 = new AWS.S3(); 
     var params = {Bucket: 'myBucket', Key: 'myKey'}; 
     s3.getSignedUrl('getObject', params, function (err, url) {  
       console.log("The URL is", url); 
     });
    

    官方文件链接 http://docs.amazonaws.cn/en_us/AWSJavaScriptSDK/guide/node-examples.html

    代码必须是这样的

    function uploadFileToS3AndGenerateUrl(cb) {
    const pass = new stream.PassThrough();//I have generated streams from file. Using this since this is what you have used. Must be a valid one.
    var params = {
                Bucket: "your-bucket", // required
                Key: key , // required
                Body: pass,
                ContentType: 'your content type',
    
            };
    s3.upload(params, function(s3Err, data) {
        if (s3Err) {
            cb(s3Err)
        }
        console.log(`File uploaded successfully at ${data.Location}`)
    
        const params = {
            Bucket: 'your-bucket',
            Key: data.key,
            Expires: 180
        };
        s3.getSignedUrl('getObject', params, (urlErr, urlData) => {
            if (urlErr) {
    
                console.log('There was an error getting your files: ' + urlErr);
                cb(urlErr);
    
            } else {
                console.log(`url: ${urlData}`);
                cb(null, urlData);
    
            }
        })
    })
    }
    
        2
  •  0
  •   IftekharDani    6 年前

    请检查我是否更新了您的代码,这可能对您有所帮助。

        /*
             * Method to upload remote file to s3
             */
            const uploadRemoteFileToS3 = async (remoteAddr) => {
                const response = await axios({
                    method: 'get',
                    url: remoteAddr,
                    responseType: 'stream'
                })
                   if(response.status===200){
                        const file_name = remoteAddr.substring(remoteAddr.lastIndexOf('/')+1);
                        const content_type = response.headers['content-type'];
                        response.data.pipe(uploadFromStream(file_name, content_type));
                    }
                    return new Promise((resolve, reject) => {
                        response.data.on('end', (response) => {
                          console.log(response)
                          resolve(response)
                        })
    
                        response.data.on('error', () => {
                          console.log(response);
                          reject(response)
                        })
                  })
            };
    
           * 
         * Method to pipe the stream 
         */
        const uploadFromStream = (file_name, content_type) => {
           return new Promise((resolve, reject) => {
              const pass = new stream.PassThrough();
              const obj_key = generateObjKey(file_name);
              const params = { Bucket: config.bucket, ACL: config.acl, Key: obj_key, ContentType: content_type, Body: pass };
              s3.upload(params, function(err, data) {
                if(!err){
                    console.log(data)
                    return resolve(data.Location);
                } else {
                    console.log(err)
                    return reject(err);
                }
              });
           });
        }
    
    //call uploadRemoteFileToS3
        uploadRemoteFileToS3(remoteAddr)
          .then((finalResponse) => {
                console.log(finalResponse)
           })
           .catch((err) => {
             console.log(err);
        });