From e0523acafe0135ead8aa9c45cdb3f45c22e97aef Mon Sep 17 00:00:00 2001 From: Yuriy Puchkov Date: Mon, 28 Sep 2020 16:20:05 +0300 Subject: [PATCH] S3 storage - amazon server side (no full documentation yet) --- .../amazon-server-side/doc/config.ini.example | 20 +++ .../s3UploaderFunction/.npmignore | 1 + .../s3UploaderFunction/app.js | 122 +++++++++++++++++ .../s3UploaderFunction/package-lock.json | 125 ++++++++++++++++++ .../s3UploaderFunction/package.json | 18 +++ .../s3UploaderFunction/testHarness.js | 25 ++++ 6 files changed, 311 insertions(+) create mode 100644 src/tools/storage/s3/amazon-server-side/doc/config.ini.example create mode 100644 src/tools/storage/s3/amazon-server-side/s3UploaderFunction/.npmignore create mode 100644 src/tools/storage/s3/amazon-server-side/s3UploaderFunction/app.js create mode 100644 src/tools/storage/s3/amazon-server-side/s3UploaderFunction/package-lock.json create mode 100644 src/tools/storage/s3/amazon-server-side/s3UploaderFunction/package.json create mode 100644 src/tools/storage/s3/amazon-server-side/s3UploaderFunction/testHarness.js diff --git a/src/tools/storage/s3/amazon-server-side/doc/config.ini.example b/src/tools/storage/s3/amazon-server-side/doc/config.ini.example new file mode 100644 index 00000000..db4fcb9d --- /dev/null +++ b/src/tools/storage/s3/amazon-server-side/doc/config.ini.example @@ -0,0 +1,20 @@ +[General] +;HTTP_PROXY_HOST=0.0.0.0 +;HTTP_PROXY_PORT=3128 + +; No authentification USER and PASSWORD should be empty +;HTTP_PROXY_USER= +;HTTP_PROXY_PASSWORD= + +HTTP_PROXY_TYPE=3 +; Proxy Types (3 is default): +; 0 Proxy is determined based on the application proxy set using setApplicationProxy() +; 1 Socks5 proxying is used +; 3 HTTP transparent proxying is used +; 4 Proxying for HTTP requests only +; 5 Proxying for FTP requests only + +[S3] +S3_URL=https://api.flameshot.org/ +S3_CREDS_URL=https://api.img.flameshot.org/ +S3_X_API_KEY=amazon-secret-key diff --git a/src/tools/storage/s3/amazon-server-side/s3UploaderFunction/.npmignore b/src/tools/storage/s3/amazon-server-side/s3UploaderFunction/.npmignore new file mode 100644 index 00000000..e7e1fb04 --- /dev/null +++ b/src/tools/storage/s3/amazon-server-side/s3UploaderFunction/.npmignore @@ -0,0 +1 @@ +tests/* diff --git a/src/tools/storage/s3/amazon-server-side/s3UploaderFunction/app.js b/src/tools/storage/s3/amazon-server-side/s3UploaderFunction/app.js new file mode 100644 index 00000000..4e5a20b7 --- /dev/null +++ b/src/tools/storage/s3/amazon-server-side/s3UploaderFunction/app.js @@ -0,0 +1,122 @@ +'use strict'; + +const uuid = require('short-uuid'); +const AWS = require('aws-sdk'); +AWS.config.update({ region: process.env.AWS_REGION || 'us-east-1' }); +const s3 = new AWS.S3(); +const cloudfront = new AWS.CloudFront(); + +const allowedImageType = 'png'; + +// Main Lambda entry point +exports.handler = async (event) => { + console.log('Event: ', event); + + let result; + if (event.resource === '/v2/image/{fileName}' && event.httpMethod === 'DELETE') { + let token = (event.headers.Authorization && event.headers.Authorization.split(' ')[1]) || ''; + result = await deleteObject(event.pathParameters.fileName, token); + if (result === false) { + return { + statusCode: 400, + isBase64Encoded: false, + headers: { + 'Access-Control-Allow-Origin': '*' + }, + body: 'Bad request' + }; + } + } else { + result = await getUploadDetails(event.resource === '/v2/image' ? 2 : 1); + } + + console.log('Result: ', result); + + return { + statusCode: 200, + isBase64Encoded: false, + headers: { + 'Access-Control-Allow-Origin': '*' + }, + body: JSON.stringify(result) + }; +}; + +const getUploadDetails = async function(version) { + const actionId = uuid.generate(); + const deleteToken = uuid.generate(); + + const s3Params = { + Bucket: process.env.UploadBucket, + Fields: { + Key: `${actionId}.${allowedImageType}`, + }, + Conditions: [ + [ 'eq', '$acl', 'private' ], + [ 'eq', '$Content-Type', `image/${allowedImageType}` ], + [ 'content-length-range', 0, 10485760 ], //allows a file size from 0B to 10 MiB + ], + Expires: 60, + }; + + if (version === 2) { + s3Params.Fields.tagging = getDeleteTags(deleteToken); + } + + console.log('getUploadURL: ', s3Params); + + const signedForm = await createPresignedPost(s3Params); + signedForm.url = `https://${process.env.UploadBucket}.s3-accelerate.amazonaws.com`; + signedForm.fields = Object.assign({ + acl: 'private', + 'Content-Type': `image/${allowedImageType}`, + }, signedForm.fields); + + const uploadDetails = { + formData: signedForm, + resultURL: `${process.env.BasePath}${s3Params.Fields.Key}` + }; + + if (version === 2) { + uploadDetails.deleteToken = deleteToken; + } + + return uploadDetails; +}; + +const deleteObject = async (key, token) => { + const s3ObjectParams = {Bucket: process.env.UploadBucket, Key: key}; + const {TagSet: tags} = await s3.getObjectTagging(s3ObjectParams).promise(); + + const storedTokenInfo = tags.find(v => v.Key === 'deleteToken'); + if (storedTokenInfo === undefined) { + console.log(`Unable to find storedTokenInfo for requested key "${key}"`); + return false; + } + + if (storedTokenInfo.Value !== token) { + console.log(`storedTokenInfo != passed token: ${storedTokenInfo.Value} != ${token}`); + return false; + } + + await s3.deleteObject(s3ObjectParams).promise(); + await cloudfront.createInvalidation({ + DistributionId: process.env.DistributionId, + InvalidationBatch: { + CallerReference: uuid.generate(), + Paths: { + Quantity: 1, + Items: [`/${key}`] + } + } + }).promise(); + + return {status: 'ok'} +}; + +const getDeleteTags = (token) => `deleteToken${token}`; + +const createPresignedPost = params => + new Promise((resolve, reject) => + s3.createPresignedPost( params, (err, data) => err ? reject(err) : resolve(data) ) + ); \ No newline at end of file diff --git a/src/tools/storage/s3/amazon-server-side/s3UploaderFunction/package-lock.json b/src/tools/storage/s3/amazon-server-side/s3UploaderFunction/package-lock.json new file mode 100644 index 00000000..667ba8f1 --- /dev/null +++ b/src/tools/storage/s3/amazon-server-side/s3UploaderFunction/package-lock.json @@ -0,0 +1,125 @@ +{ + "name": "none", + "version": "1.0.0", + "lockfileVersion": 1, + "requires": true, + "dependencies": { + "any-base": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/any-base/-/any-base-1.1.0.tgz", + "integrity": "sha512-uMgjozySS8adZZYePpaWs8cxB9/kdzmpX6SgJZ+wbz1K5eYk5QMYDVJaZKhxyIHUdnnJkfR7SVgStgH7LkGUyg==" + }, + "aws-sdk": { + "version": "2.700.0", + "resolved": "https://registry.npmjs.org/aws-sdk/-/aws-sdk-2.700.0.tgz", + "integrity": "sha512-faBkr/D3IavfL2mwst4/thiKsHkN8YCwU9927Mkiushbe7n4UXxlcNf7LnVxFyjr3WIf4KfziSw4bajRAiAjYA==", + "requires": { + "buffer": "4.9.2", + "events": "1.1.1", + "ieee754": "1.1.13", + "jmespath": "0.15.0", + "querystring": "0.2.0", + "sax": "1.2.1", + "url": "0.10.3", + "uuid": "3.3.2", + "xml2js": "0.4.19" + }, + "dependencies": { + "uuid": { + "version": "3.3.2", + "resolved": "https://registry.npmjs.org/uuid/-/uuid-3.3.2.tgz", + "integrity": "sha512-yXJmeNaw3DnnKAOKJE51sL/ZaYfWJRl1pK9dr19YFCu0ObS231AB1/LbqTKRAQ5kw8A90rA6fr4riOUpTZvQZA==" + } + } + }, + "base64-js": { + "version": "1.3.1", + "resolved": "https://registry.npmjs.org/base64-js/-/base64-js-1.3.1.tgz", + "integrity": "sha512-mLQ4i2QO1ytvGWFWmcngKO//JXAQueZvwEKtjgQFM4jIK0kU+ytMfplL8j+n5mspOfjHwoAg+9yhb7BwAHm36g==" + }, + "buffer": { + "version": "4.9.2", + "resolved": "https://registry.npmjs.org/buffer/-/buffer-4.9.2.tgz", + "integrity": "sha512-xq+q3SRMOxGivLhBNaUdC64hDTQwejJ+H0T/NB1XMtTVEwNTrfFF3gAxiyW0Bu/xWEGhjVKgUcMhCrUy2+uCWg==", + "requires": { + "base64-js": "^1.0.2", + "ieee754": "^1.1.4", + "isarray": "^1.0.0" + } + }, + "events": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/events/-/events-1.1.1.tgz", + "integrity": "sha1-nr23Y1rQmccNzEwqH1AEKI6L2SQ=" + }, + "ieee754": { + "version": "1.1.13", + "resolved": "https://registry.npmjs.org/ieee754/-/ieee754-1.1.13.tgz", + "integrity": "sha512-4vf7I2LYV/HaWerSo3XmlMkp5eZ83i+/CDluXi/IGTs/O1sejBNhTtnxzmRZfvOUqj7lZjqHkeTvpgSFDlWZTg==" + }, + "isarray": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/isarray/-/isarray-1.0.0.tgz", + "integrity": "sha1-u5NdSFgsuhaMBoNJV6VKPgcSTxE=" + }, + "jmespath": { + "version": "0.15.0", + "resolved": "https://registry.npmjs.org/jmespath/-/jmespath-0.15.0.tgz", + "integrity": "sha1-o/Iiqarp+Wb10nx5ZRDigJF2Qhc=" + }, + "punycode": { + "version": "1.3.2", + "resolved": "https://registry.npmjs.org/punycode/-/punycode-1.3.2.tgz", + "integrity": "sha1-llOgNvt8HuQjQvIyXM7v6jkmxI0=" + }, + "querystring": { + "version": "0.2.0", + "resolved": "https://registry.npmjs.org/querystring/-/querystring-0.2.0.tgz", + "integrity": "sha1-sgmEkgO7Jd+CDadW50cAWHhSFiA=" + }, + "sax": { + "version": "1.2.1", + "resolved": "https://registry.npmjs.org/sax/-/sax-1.2.1.tgz", + "integrity": "sha1-e45lYZCyKOgaZq6nSEgNgozS03o=" + }, + "short-uuid": { + "version": "3.1.1", + "resolved": "https://registry.npmjs.org/short-uuid/-/short-uuid-3.1.1.tgz", + "integrity": "sha512-7dI69xtJYpTIbg44R6JSgrbDtZFuZ9vAwwmnF/L0PinykbFrhQ7V8omKsQcVw1TP0nYJ7uQp1PN6/aVMkzQFGQ==", + "requires": { + "any-base": "^1.1.0", + "uuid": "^3.3.2" + }, + "dependencies": { + "uuid": { + "version": "3.4.0", + "resolved": "https://registry.npmjs.org/uuid/-/uuid-3.4.0.tgz", + "integrity": "sha512-HjSDRw6gZE5JMggctHBcjVak08+KEVhSIiDzFnT9S9aegmp85S/bReBVTb4QTFaRNptJ9kuYaNhnbNEOkbKb/A==" + } + } + }, + "url": { + "version": "0.10.3", + "resolved": "https://registry.npmjs.org/url/-/url-0.10.3.tgz", + "integrity": "sha1-Ah5NnHcF8hu/N9A861h2dAJ3TGQ=", + "requires": { + "punycode": "1.3.2", + "querystring": "0.2.0" + } + }, + "xml2js": { + "version": "0.4.19", + "resolved": "https://registry.npmjs.org/xml2js/-/xml2js-0.4.19.tgz", + "integrity": "sha512-esZnJZJOiJR9wWKMyuvSE1y6Dq5LCuJanqhxslH2bxM6duahNZ+HMpCLhBQGZkbX6xRf8x1Y2eJlgt2q3qo49Q==", + "requires": { + "sax": ">=0.6.0", + "xmlbuilder": "~9.0.1" + } + }, + "xmlbuilder": { + "version": "9.0.7", + "resolved": "https://registry.npmjs.org/xmlbuilder/-/xmlbuilder-9.0.7.tgz", + "integrity": "sha1-Ey7mPS7FVlxVfiD0wi35rKaGsQ0=" + } + } +} diff --git a/src/tools/storage/s3/amazon-server-side/s3UploaderFunction/package.json b/src/tools/storage/s3/amazon-server-side/s3UploaderFunction/package.json new file mode 100644 index 00000000..c91f6e1c --- /dev/null +++ b/src/tools/storage/s3/amazon-server-side/s3UploaderFunction/package.json @@ -0,0 +1,18 @@ +{ + "name": "none", + "private": true, + "version": "1.0.0", + "description": "", + "main": "app.js", + "scripts": { + "start": "node testHarness.js", + "pack": "zip -r ../.infrastructure/lambda.zip ./", + "test": "echo \"Error: no test specified\" && exit 1" + }, + "keywords": [], + "license": "UNLICENSED", + "dependencies": { + "aws-sdk": "^2.478.0", + "short-uuid": "^3.1.1" + } +} diff --git a/src/tools/storage/s3/amazon-server-side/s3UploaderFunction/testHarness.js b/src/tools/storage/s3/amazon-server-side/s3UploaderFunction/testHarness.js new file mode 100644 index 00000000..2b8c9236 --- /dev/null +++ b/src/tools/storage/s3/amazon-server-side/s3UploaderFunction/testHarness.js @@ -0,0 +1,25 @@ +/* + Copyright 2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. + Permission is hereby granted, free of charge, to any person obtaining a copy of this + software and associated documentation files (the "Software"), to deal in the Software + without restriction, including without limitation the rights to use, copy, modify, + merge, publish, distribute, sublicense, and/or sell copies of the Software, and to + permit persons to whom the Software is furnished to do so. + THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, + INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A + PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT + HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION + OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE + SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +*/ + +// For local testing + +const { handler } = require('./app') + +const main = async () => { + await handler() +} + +main() +