Share
Embed
Testable Lambda: Working Effectively with Legacy Lambda
AWS Dev Day Tokyo 2017 Day4 Track1 15:20 - 16:00
#AWSSummit #testlambda
当日のセッション録画:
Testable Lambda|AWS Summit Tokyo 2017 - YouTube
https://www.youtube.com/watch?v=C0zNc4bdWhY
Takuto WADA PRO
June 02, 2017
Transcript
69 slides
-
// dependencies var async = require('async'); var AWS = require('aws-sdk');
var gm = require('gm') .subClass({ imageMagick: true }); // Enable ImageMagick integration. var util = require('util'); // constants var MAX_WIDTH = 100; var MAX_HEIGHT = 100; // get reference to S3 client var s3 = new AWS.S3(); exports.handler = function(event, context, callback) { // Read options from the event. console.log("Reading options from event:\n", util.inspect(event, {depth: 5})); var srcBucket = event.Records[0].s3.bucket.name; // Object key may have spaces or unicode non-ASCII characters. var srcKey = decodeURIComponent(event.Records[0].s3.object.key.replace(/\+/g, " ")); var dstBucket = srcBucket + "resized"; var dstKey = "resized-" + srcKey; // Sanity check: validate that source and destination are different buckets. if (srcBucket == dstBucket) { callback("Source and destination buckets are the same."); return; } // Infer the image type. var typeMatch = srcKey.match(/\.([^.]*)$/); if (!typeMatch) { callback("Could not determine the image type."); return; } var imageType = typeMatch[1]; if (imageType != "jpg" && imageType != "png") { callback('Unsupported image type: ${imageType}'); return; } // Download the image from S3, transform, and upload to a different S3 bucket. async.waterfall([ function download(next) { // Download the image from S3 into a buffer. s3.getObject({ Bucket: srcBucket, Key: srcKey }, next); }, function transform(response, next) { gm(response.Body).size(function(err, size) { // Infer the scaling factor to avoid stretching the image unnaturally. var scalingFactor = Math.min( MAX_WIDTH / size.width, MAX_HEIGHT / size.height ); var width = scalingFactor * size.width; var height = scalingFactor * size.height; // Transform the image buffer in memory. this.resize(width, height) .toBuffer(imageType, function(err, buffer) { if (err) { next(err); } else { next(null, response.ContentType, buffer); } }); }); }, function upload(contentType, data, next) { // Stream the transformed image to a different S3 bucket. s3.putObject({ Bucket: dstBucket, Key: dstKey, Body: data, ContentType: contentType }, next); } ], function (err) { if (err) { console.error( 'Unable to resize ' + srcBucket + '/' + srcKey + ' and upload to ' + dstBucket + '/' + dstKey + ' due to an error: ' + err ); } else { console.log( 'Successfully resized ' + srcBucket + '/' + srcKey + ' and uploaded to ' + dstBucket + '/' + dstKey ); } callback(null, "message"); } ); }; w ߦͷίʔυɺͭͷؔ w ݹ͍ίʔσΟϯάελΠϧ &4 BTZOD w Ͳ͏Βσουίʔυ͕͋Δ w ؍ଌ͢Δҙຯ߹͍ͷগͳ͍Γ w ˠࣗಈςετΛॻ͍ͯৼΔ͍͕มΘͬͯͳ͍ ͜ͱΛ͔֬Ίͳ͕Βɺ։ൃܧଓʹඋ͑ͯ৽͍͠ ίʔσΟϯάελΠϧ &4 1SPNJTF ʹҠ ߦ͍ͨ͠ -
// dependencies var async = require('async'); var AWS = require('aws-sdk');
var gm = require('gm') .subClass({ imageMagick: true }); // Enable ImageMagick integration. var util = require('util'); // constants var MAX_WIDTH = 100; var MAX_HEIGHT = 100; // get reference to S3 client var s3 = new AWS.S3(); exports.handler = function(event, context, callback) { // Read options from the event. console.log("Reading options from event:\n", util.inspect(event, {depth: 5})); var srcBucket = event.Records[0].s3.bucket.name; // Object key may have spaces or unicode non-ASCII characters. var srcKey = decodeURIComponent(event.Records[0].s3.object.key.replace(/\+/g, " ")); var dstBucket = srcBucket + "resized"; var dstKey = "resized-" + srcKey; -
var dstBucket = srcBucket + "resized"; var dstKey = "resized-"
+ srcKey; // Sanity check: validate that source and destination are different buckets. if (srcBucket == dstBucket) { callback("Source and destination buckets are the same."); return; } // Infer the image type. var typeMatch = srcKey.match(/\.([^.]*)$/); if (!typeMatch) { callback("Could not determine the image type."); return; } var imageType = typeMatch[1]; if (imageType != "jpg" && imageType != "png") { callback('Unsupported image type: ${imageType}'); return; } // Download the image from S3, transform, and upload to a different S3 bucket. async.waterfall([ function download(next) { // Download the image from S3 into a buffer. -
// Download the image from S3, transform, and upload to
a different S3 bucket. async.waterfall([ function download(next) { // Download the image from S3 into a buffer. s3.getObject({ Bucket: srcBucket, Key: srcKey }, next); }, function transform(response, next) { gm(response.Body).size(function(err, size) { // Infer the scaling factor to avoid stretching the image unnaturally. var scalingFactor = Math.min( MAX_WIDTH / size.width, MAX_HEIGHT / size.height ); var width = scalingFactor * size.width; var height = scalingFactor * size.height; // Transform the image buffer in memory. this.resize(width, height) .toBuffer(imageType, function(err, buffer) { if (err) { next(err); -
// Transform the image buffer in memory. this.resize(width, height) .toBuffer(imageType,
function(err, buffer) { if (err) { next(err); } else { next(null, response.ContentType, buffer); } }); }); }, function upload(contentType, data, next) { // Stream the transformed image to a different S3 bucket. s3.putObject({ Bucket: dstBucket, Key: dstKey, Body: data, ContentType: contentType }, next); } ], function (err) { if (err) { console.error( -
Bucket: dstBucket, Key: dstKey, Body: data, ContentType: contentType }, next);
} ], function (err) { if (err) { console.error( 'Unable to resize ' + srcBucket + '/' + srcKey + ' and upload to ' + dstBucket + '/' + dstKey + ' due to an error: ' + err ); } else { console.log( 'Successfully resized ' + srcBucket + '/' + srcKey + ' and uploaded to ' + dstBucket + '/' + dstKey ); } callback(null, "message"); } ); }; -
ςετखॱॻWFS w 4όέοτ४උ w TPVSDFόέοτͱEFTUJOBUJPOόέοτΛ࡞ w TPVSDFόέοτʹαϯϓϧΦϒδΣΫτΛΞοϓϩʔυ͓ͯ͘͠ w -BNCEBؔΛ࡞ͯ͠σϓϩΠ w
σϓϩΠύοέʔδΛ࡞ w OQNJOTUBMMBTZODHN w σΟϨΫτϦؙ͝ͱ;*1ѹॖ w ࣮ߦϩʔϧΛ࡞ w *".ίϯιʔϧͰ<3PMF/BNF> <4FMFDU3PMF5ZQF> <"UUBDI1PMJDZ>Λઃఆ w ϩʔϧͷ"3/ΛϝϞ w -BNCEBؔΛσϓϩΠ w BXTMBNCEBDSFBUFGVODUJPO w ؔͷ"3/ΛϝϞ w ࢹͰςετ w "NB[PO4αϯϓϧΠϕϯτσʔλΛϑΝΠϧʹอଘ w -BNCEB$-*JOWPLFίϚϯυΛ࣮ߦͯؔ͠Λݺͼग़͠ w αϜωΠϧ͕λʔήοτόέοτʹ࡞͞Εͨ͜ͱΛ֬ೝ w "84-BNCEBίϯιʔϧͰ-BNCEBؔͷϩάΛ֬ೝ ॴཁ࣌ؒ -
AWSTemplateFormatVersion: '2010-09-09' Transform: AWS::Serverless-2016-10-31 Description: Image resize example Resources: SourceBucket:
Type: AWS::S3::Bucket Properties: BucketName: images DestinationBucket: Type: AWS::S3::Bucket Properties: BucketName: imagesresized ProcessorFunction: Type: AWS::Serverless::Function Properties: Handler: index.handler Runtime: nodejs6.10 CodeUri: ./code Policies: AmazonS3FullAccess Events: ImageUpload: Type: S3 Properties: Bucket: !Ref SourceBucket Events: s3:ObjectCreated:* ςετखॱॻWFS w 4".ఆٛϑΝΠϧ࡞ w 4".σϓϩΠ༻4όέοτ४උ w -BNCEBؔΛ࡞ͯ͠σϓϩΠ w BXTDMPVEGPSNBUJPOQBDLBHF w BXTDMPVEGPSNBUJPOEFQMPZ w BXTDMJͰςετ w BXTMPHTHFUBXTMBNCEB<MPHJE>XBUDI w BXTTDQIPHFQOHTJNBHFTIPHFQOH w BXTTMTJNBHFTSFTJ[FE ॴཁ࣌ؒ -
$ SERVICES=s3 make infra . .venv/bin/activate; PYTHONPATH=. exec localstack/mock/infra.py Starting
local dev environment. CTRL-C to quit. Starting mock S3 (port 4572)... Ready. $ aws --endpoint-url=http://localhost:4572 s3 ls $ aws --endpoint-url=http://localhost:4572 s3 mb s3://my-test-bucket make_bucket: my-test-bucket $ aws --endpoint-url=http://localhost:4572 s3 ls 2006-02-04 01:45:09 my-test-bucket $ aws --endpoint-url=http://localhost:4572 s3 cp ~/hoge.png s3://my-test-bucket upload: hoge.png to s3://my-test-bucket/hoge.png $ aws --endpoint-url=http://localhost:4572 s3 ls s3://my-test-bucket 2017-06-02 11:39:22 7538 hoge.png MPDBMTUBDLΛNBLFEPDLFSͰಈ͔͢ endpoint-url を指定すれば aws cli すら欺ける -
const assert = require('assert'); const fs = require('fs'); const path
= require('path'); const AWS = require('aws-sdk'); const s3 = new AWS.S3({ s3ForcePathStyle: true, logger: console, endpoint: new AWS.Endpoint('http://localhost:4572') }); describe('localstack learning', () => { before(() => { return s3.createBucket({Bucket: 'test-bucket'}).promise().then(() => { return s3.putObject({ Bucket: 'test-bucket', Key: 'TQ_LOGO.png', ContentType: 'image/png', Body: fs.readFileSync(path.join(__dirname, '..', 'fixtures', 'TQ_LOGO.png')) }).promise(); }); }); it('s3.getObject', () => { return s3.getObject({Bucket: 'test-bucket', Key: 'TQ_LOGO.png'}).promise().then((res) => { assert(res); }); }); }); MPDBMTUBDLͱ1SPNJTFͷֶशςετΛॻ͍ͯΈΔ 学習テスト: ライブラリ等の 学習のみに特化したテスト -
const assert = require('assert'); const fs = require('fs'); const path
= require('path'); const AWS = require('aws-sdk'); const s3 = new AWS.S3({ s3ForcePathStyle: true, logger: console, endpoint: new AWS.Endpoint('http://localhost:4572') }); describe('localstack learning', () => { before(() => { return s3.createBucket({Bucket: 'test-bucket'}).promise().then(() => { return s3.putObject({ Bucket: 'test-bucket', Key: 'TQ_LOGO.png', ContentType: 'image/png', Body: fs.readFileSync(path.join(__dirname, '..', 'fixtures', 'TQ_LOGO.png')) }).promise(); }); }); it('s3.getObject', () => { return s3.getObject({Bucket: 'test-bucket', Key: 'TQ_LOGO.png'}).promise().then((res) => { assert(res); }); }); }); endpoint を localstack に 向けるだけ! S3 のバケット URL をパス形式に変えておく (ハマりポイント) -
const assert = require('assert'); const fs = require('fs'); const path
= require('path'); const AWS = require('aws-sdk'); const s3 = new AWS.S3({ s3ForcePathStyle: true, logger: console, endpoint: new AWS.Endpoint('http://localhost:4572') }); describe('localstack learning', () => { before(() => { return s3.createBucket({Bucket: 'test-bucket'}).promise().then(() => { return s3.putObject({ Bucket: 'test-bucket', Key: 'TQ_LOGO.png', ContentType: 'image/png', Body: fs.readFileSync(path.join(__dirname, '..', 'fixtures', 'TQ_LOGO.png')) }).promise(); }); }); it('s3.getObject', () => { return s3.getObject({Bucket: 'test-bucket', Key: 'TQ_LOGO.png'}).promise().then((res) => { assert(res); }); }); }); aws-sdk の Promise 機能をモリモリ試す -
var MAX_HEIGHT = 100; // get reference to S3 client
var s3; if (process.env.NODE_ENV === 'production') { s3 = new AWS.S3(); } else { s3 = new AWS.S3({ s3ForcePathStyle: true, logger: console, endpoint: new AWS.Endpoint('http://localhost:4572') }); } exports.handler = function(event, context, callback) { // Read options from the event. Ξϯνύλʔϯ5FTU-PHJDJO1SPEVDUJPO # " % -
var AWS = require('aws-sdk'); var s3 = new AWS.S3(); var
onObjectCreated = require('./on-object-created'); exports.handler = function(event, context, callback) { onObjectCreated({s3, event, callback}); }; ॲཧຊମΛผϞδϡʔϧʹग़͠ɺҾΛ߹෦ʹ͢Δ ( 0 0 % // dependencies var async = require('async'); var gm = require('gm') .subClass({ imageMagick: true }); // Enable ImageMagick integration. var util = require('util'); // constants var MAX_WIDTH = 100; var MAX_HEIGHT = 100; module.exports = function({s3, event, callback}) { // Read options from the event. console.log("Reading options from event:\n", util.inspect(event, {depth: 5})); var srcBucket = event.Records[0].s3.bucket.name; ৽نϑΝΠϧPOPCKFDUDSFBUFEKT JOEFYKT -
const assert = require('assert'); const AWS = require('aws-sdk'); const s3
= new AWS.S3({s3ForcePathStyle: true, endpoint: new AWS.Endpoint('http://localhost:4572')}); const onObjectCreated = require('../on-object-created'); describe('localstack based test suite for happy path cases', () => { let now, event; beforeEach(() => { now = new Date().getTime(); event = { Records: [ { s3: { bucket: { name: `test-bucket-${now}` }, object: { key: `TQ_LOGO_${now}.png` } } } ] }; return s3.createBucket({Bucket: `test-bucket-${now}`}).promise() .then(() => s3.createBucket({Bucket: `test-bucket-${now}resized`}).promise()) .then(() => s3.putObject({ Bucket: `test-bucket-${now}`, Key: `TQ_LOGO_${now}.png`, ContentType: 'image/png', Body: fs.readFileSync(path.join(__dirname, '..', 'fixtures', 'TQ_LOGO.png')) }).promise()); }); it('onObjectCreated callback', () => { return new Promise((resolve, reject) => { const callback = (err, message) => err ? reject(err) : resolve(message); onObjectCreated({s3, event, callback}); }).then((message) => { assert(/^message/.test(message)); }); }); ͜ΕͰςετ͕ॻ͚Δ -
const assert = require('assert'); const AWS = require('aws-sdk'); const s3
= new AWS.S3({s3ForcePathStyle: true, endpoint: new AWS.Endpoint('http://localhost:4572')}); const onObjectCreated = require('../on-object-created'); describe('localstack based test suite for happy path cases', () => { let now, event; beforeEach(() => { now = new Date().getTime(); event = { Records: [ { s3: { bucket: { name: `test-bucket-${now}` }, object: { key: `TQ_LOGO_${now}.png` } } } ] }; return s3.createBucket({Bucket: `test-bucket-${now}`}).promise() .then(() => s3.createBucket({Bucket: `test-bucket-${now}resized`}).promise()) .then(() => s3.putObject({ Bucket: `test-bucket-${now}`, Key: `TQ_LOGO_${now}.png`, ContentType: 'image/png', Body: fs.readFileSync(path.join(__dirname, '..', 'fixtures', 'TQ_LOGO.png')) }).promise()); }); it('onObjectCreated callback', () => { return new Promise((resolve, reject) => { const callback = (err, message) => err ? reject(err) : resolve(message); onObjectCreated({s3, event, callback}); }).then((message) => { assert(/^message/.test(message)); }); }); ߹෦Λ׆༻ͯ͠ςετΛॻ͘ -
Body: fs.readFileSync(path.join(__dirname, '..', 'fixtures', 'TQ_LOGO.png')) }).promise()); }); it('onObjectCreated callback', ()
=> { return new Promise((resolve, reject) => { const callback = (err, message) => err ? reject(err) : resolve(message); onObjectCreated({s3, event, callback}); }).then((message) => { assert(/^message/.test(message)); }); }); it('onObjectCreated creates and puts thumbnail into destination bucket', () => { return new Promise((resolve, reject) => { const callback = (err, message) => err ? reject(err) : resolve(message); onObjectCreated({s3, event, callback}); }) .then(() => { return s3.waitFor('objectExists', { Bucket: `test-bucket-${now}resized`, Key: `resized-TQ_LOGO_${now}.png` }).promise(); }) .then((data) => { assert(data); }); }); }); -
module.exports = function({s3, event, callback}) { // Read options from
the event. console.log("Reading options from event:\n", util.inspect(event, {depth: 5})); var srcBucket = event.Records[0].s3.bucket.name; // Object key may have spaces or unicode non-ASCII characters. var srcKey = decodeURIComponent(event.Records[0].s3.object.key.replace(/\+/g, " ")); var dstBucket = srcBucket + "resized"; var dstKey = "resized-" + srcKey; // Sanity check: validate that source and destination are different buckets. if (srcBucket == dstBucket) { callback("Source and destination buckets are the same."); return; } // Infer the image type. var typeMatch = srcKey.match(/\.([^.]*)$/); if (!typeMatch) { callback("Could not determine the image type."); return; } var imageType = typeMatch[1]; if (imageType != "jpg" && imageType != "png") { callback('Unsupported image type: ${imageType}'); return; } IUUQTHJUIVCDPNUXBEBDPWFSMBZFM ݱঢ়֬ೝͷͨΊΧόϨοδଌఆ -
IUUQTHJUIVCDPNUXBEBDPWFSMBZFM // Transform the image buffer in memory. this.resize(width, height)
.toBuffer(imageType, function(err, buffer) { if (err) { next(err); } else { next(null, response.ContentType, buffer); } }); }); }, function upload(contentType, data, next) { // Stream the transformed image to a different S3 bucket. s3.putObject({ Bucket: dstBucket, Key: dstKey, Body: data, ContentType: contentType }, next); } ], function (err) { if (err) { console.error( 'Unable to resize ' + srcBucket + '/' + srcKey + ' and upload to ' + dstBucket + '/' + dstKey + ' due to an error: ' + err ); ྫ֎ܥ͕ςετ͞Εͯͳ͍͜ͱ͕Θ͔Δ -
const assert = require('assert'); const onObjectCreated = require('../on-object-created'); describe('small sized
test suite for exceptional cases:', () => { it('file without extension', () => { const event = { Records: [ { s3: { bucket: { name: `test-bucket` }, object: { key: `TQ_LOGO` } } } ] }; onObjectCreated({event, callback: (err, message) => { assert(err === 'Could not determine the image type.'); }}); }); it('extension is not `.png` nor `.jpg`', () => { const event = { Records: [ { s3: { bucket: { name: `test-bucket` }, object: { key: `TQ_LOGO.txt` } } } ] }; onObjectCreated({event, callback: (err, message) => { assert(err === 'Unsupported image type: txt'); }}); }); }); MPDBMTUBDL͢ΒΘͳ͍TNBMMUFTUͰྫ֎ܥΛςετ͢Δ -
IUUQTHJUIVCDPNQPXFSBTTFSUKTQPXFSBTTFSU > mocha --require intelli-espower-loader 'test/small/**/*.js' small sized test suite
for exceptional cases: ✓ file without extension 1) extension is not `.png` nor `.jpg` 1 passing (47ms) 1 failing 1) small sized test suite for exceptional cases: extension is not `.png` nor `.jpg`: AssertionError: # test/small/exceptional_cases_test.js:25 assert(err === 'Unsupported image type: txt') | | | false "Unsupported image type: ${imageType}" --- [string] 'Unsupported image type: txt' +++ [string] err @@ -21,7 +21,16 @@ pe: -txt +${imageType} _人人人人人人人人人人_ > 突然のテスト失敗 <  ̄YYYYYYYYYY ̄ -
IUUQTHJUIVCDPNQPXFSBTTFSUKTQPXFSBTTFSU > mocha --require intelli-espower-loader 'test/small/**/*.js' small sized test suite
for exceptional cases: ✓ file without extension 1) extension is not `.png` nor `.jpg` 1 passing (47ms) 1 failing 1) small sized test suite for exceptional cases: extension is not `.png` nor `.jpg`: AssertionError: # test/small/exceptional_cases_test.js:25 assert(err === 'Unsupported image type: txt') | | | false "Unsupported image type: ${imageType}" --- [string] 'Unsupported image type: txt' +++ [string] err @@ -21,7 +21,16 @@ pe: -txt +${imageType} ڭ܇ςετͯ͠ͳ͍ίʔυಈ͔ͳ͍ if (imageType != "jpg" && imageType != "png") { callback('Unsupported image type: ${imageType}'); return; } -
AWSTemplateFormatVersion: '2010-09-09' Transform: AWS::Serverless-2016-10-31 Description: Image resize example Resources: SourceBucket:
Type: AWS::S3::Bucket Properties: BucketName: test-{{branchName}} DestinationBucket: Type: AWS::S3::Bucket Properties: BucketName: test-{{branchName}}resized ProcessorFunction: Type: AWS::Serverless::Function Properties: Handler: index.handler Runtime: nodejs6.10 CodeUri: ./code Policies: AmazonS3FullAccess Events: ImageUpload: Type: S3 Properties: Bucket: !Ref SourceBucket Events: s3:ObjectCreated:* ϒϥϯνຖʹҟͳΔڥΛ࡞Δ 手に馴染んだテンプレートエンジンを使って SAM 定義をブランチ毎に生成する -
const assert = require('assert'); const fs = require('fs'); const path
= require('path'); const AWS = require('aws-sdk'); const s3 = new AWS.S3({ logger: console }); const execSync = require('child_process').execSync; const branch = execSync('git rev-parse --abbrev-ref @').toString().replace(/\n$/, ''); describe('large sized test suite', () => { it('happy path', () => { const now = new Date().getTime(); return Promise.resolve() .then(() => { return s3.putObject({ Bucket: `test-${branch}`, Key: `TQ_LOGO_${now}.png`, Body: fs.readFileSync(path.join(__dirname, '..', 'fixtures', 'TQ_LOGO.png')), ContentType: 'image/png' }).promise(); }) .then(() => { return s3.waitFor('objectExists', { Bucket: `test-${branch}resized`, Key: `resized-TQ_LOGO_${now}.png` }).promise(); }) .then((data) => { assert(data); }); }); }); MBSHFϒϥϯνผͷ"84ڥʹର͢Δςετ -
const gm = require('gm').subClass({ imageMagick: true }); const util =
require('util'); const resize = require('./resize'); module.exports = function onObjectCreated ({s3, event, callback}) { console.log('Reading options from event:\n', util.inspect(event, {depth: 5})); const srcBucket = event.Records[0].s3.bucket.name; const srcKey = decodeURIComponent(event.Records[0].s3.object.key.replace(/\+/g, ' ')); const dstBucket = srcBucket + 'resized'; const dstKey = 'resized-' + srcKey; resize({s3, gm, srcBucket, srcKey, dstBucket, dstKey}) .then((message) => { console.log(message); callback(null, message); }) .catch((err) => { const message = `Unable to resize ${srcBucket}/${srcKey} and upload to ${dstBucket}/ ${dstKey} due to an error: ${err}`; console.error(message); callback(null, message); }); }; "84-BNCEBʹґଘͨ͠ཁૉΛઙ͍֊ͰҾ͖ണ͕͢