Last active
May 23, 2024 21:44
-
-
Save amiantos/16bacc9ed742c91151fcf1a41012445e to your computer and use it in GitHub Desktop.
Zip Multiple Files from S3 using AWS Lambda Function
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| // Lambda S3 Zipper | |
| // http://amiantos.net/zip-multiple-files-on-aws-s3/ | |
| // | |
| // Accepts a bundle of data in the format... | |
| // { | |
| // "bucket": "your-bucket", | |
| // "destination_key": "zips/test.zip", | |
| // "files": [ | |
| // { | |
| // "uri": "...", (options: S3 file key or URL) | |
| // "filename": "...", (filename of file inside zip) | |
| // "type": "..." (options: [file, url]) | |
| // } | |
| // ] | |
| // } | |
| // Saves zip file at "destination_key" location | |
| "use strict"; | |
| const AWS = require("aws-sdk"); | |
| const awsOptions = { | |
| region: "us-east-1", | |
| httpOptions: { | |
| timeout: 300000 // Matching Lambda function timeout | |
| } | |
| }; | |
| const s3 = new AWS.S3(awsOptions); | |
| const archiver = require("archiver"); | |
| const stream = require("stream"); | |
| const request = require("request"); | |
| const streamTo = (bucket, key, resolve) => { | |
| var passthrough = new stream.PassThrough(); | |
| s3.upload( | |
| { | |
| Bucket: bucket, | |
| Key: key, | |
| Body: passthrough, | |
| ContentType: "application/zip", | |
| ServerSideEncryption: "AES256" | |
| }, | |
| (err, data) => { | |
| if (err) throw err; | |
| console.log("Zip uploaded"); | |
| resolve(); | |
| } | |
| ).on("httpUploadProgress", (progress) => { | |
| console.log(progress); | |
| }); | |
| return passthrough; | |
| }; | |
| // Kudos to this person on GitHub for this getStream solution | |
| // https://github.com/aws/aws-sdk-js/issues/2087#issuecomment-474722151 | |
| const getStream = (bucket, key) => { | |
| let streamCreated = false; | |
| const passThroughStream = new stream.PassThrough(); | |
| passThroughStream.on("newListener", event => { | |
| if (!streamCreated && event == "data") { | |
| const s3Stream = s3 | |
| .getObject({ Bucket: bucket, Key: key }) | |
| .createReadStream(); | |
| s3Stream | |
| .on("error", err => passThroughStream.emit("error", err)) | |
| .pipe(passThroughStream); | |
| streamCreated = true; | |
| } | |
| }); | |
| return passThroughStream; | |
| }; | |
| exports.handler = async (event, context, callback) => { | |
| var bucket = event["bucket"]; | |
| var destinationKey = event["destination_key"]; | |
| var files = event["files"]; | |
| await new Promise(async (resolve, reject) => { | |
| var zipStream = streamTo(bucket, destinationKey, resolve); | |
| zipStream.on("error", reject); | |
| var archive = archiver("zip"); | |
| archive.on("error", err => { | |
| throw new Error(err); | |
| }); | |
| archive.pipe(zipStream); | |
| for (const file of files) { | |
| if (file["type"] == "file") { | |
| archive.append(getStream(bucket, file["uri"]), { | |
| name: file["filename"] | |
| }); | |
| } else if (file["type"] == "url") { | |
| archive.append(request(file["uri"]), { name: file["filename"] }); | |
| } | |
| } | |
| archive.finalize(); | |
| }).catch(err => { | |
| throw new Error(err); | |
| }); | |
| callback(null, { | |
| statusCode: 200, | |
| body: { final_destination: destinationKey } | |
| }); | |
| }; |
Hi, I need someone to help me please. I have created a Lambda function to trigger S3 bucket and zip the uploaded files then store them in a destination bucket. but when I upload files it doesn't appear to destination bucket. I have searched a lot but couldn't find any solution for that. I configured all the permission and IAM but still it doesn't work.
zipkit.io is another way to solve this problem.
Really happy to see how this gist has grown and evolved over the years. Kudos to everyone who has weighed in, offered more code and advice!
I am going to join in and sing praises for @amiantos - you saved my butt today!
I had to write a Lambda to zip thousands of files. The task would just go dark - no zip, no errors. Your V3 script did the trick - well done!
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment
I was struggling with this today. Glad to know that I am not alone <3