$30 off During Our Annual Pro Sale. View Details »

Testable Lambda: Working Effectively with Legac...

Testable Lambda: Working Effectively with Legacy Lambda

AWS Dev Day Tokyo 2017 Day4 Track1 15:20 - 16:00

#AWSSummit #testlambda

当日のセッション録画:
Testable Lambda|AWS Summit Tokyo 2017 - YouTube
https://www.youtube.com/watch?v=C0zNc4bdWhY

Takuto Wada

June 02, 2017
Tweet

More Decks by Takuto Wada

Other Decks in Programming

Transcript

  1. // dependencies var async = require('async'); var AWS = require('aws-sdk');

    var gm = require('gm') .subClass({ imageMagick: true }); // Enable ImageMagick integration. var util = require('util'); // constants var MAX_WIDTH = 100; var MAX_HEIGHT = 100; // get reference to S3 client var s3 = new AWS.S3(); exports.handler = function(event, context, callback) { // Read options from the event. console.log("Reading options from event:\n", util.inspect(event, {depth: 5})); var srcBucket = event.Records[0].s3.bucket.name; // Object key may have spaces or unicode non-ASCII characters. var srcKey = decodeURIComponent(event.Records[0].s3.object.key.replace(/\+/g, " ")); var dstBucket = srcBucket + "resized"; var dstKey = "resized-" + srcKey; // Sanity check: validate that source and destination are different buckets. if (srcBucket == dstBucket) { callback("Source and destination buckets are the same."); return; } // Infer the image type. var typeMatch = srcKey.match(/\.([^.]*)$/); if (!typeMatch) { callback("Could not determine the image type."); return; } var imageType = typeMatch[1]; if (imageType != "jpg" && imageType != "png") { callback('Unsupported image type: ${imageType}'); return; } // Download the image from S3, transform, and upload to a different S3 bucket. async.waterfall([ function download(next) { // Download the image from S3 into a buffer. s3.getObject({ Bucket: srcBucket, Key: srcKey }, next); }, function transform(response, next) { gm(response.Body).size(function(err, size) { // Infer the scaling factor to avoid stretching the image unnaturally. var scalingFactor = Math.min( MAX_WIDTH / size.width, MAX_HEIGHT / size.height ); var width = scalingFactor * size.width; var height = scalingFactor * size.height; // Transform the image buffer in memory. this.resize(width, height) .toBuffer(imageType, function(err, buffer) { if (err) { next(err); } else { next(null, response.ContentType, buffer); } }); }); }, function upload(contentType, data, next) { // Stream the transformed image to a different S3 bucket. s3.putObject({ Bucket: dstBucket, Key: dstKey, Body: data, ContentType: contentType }, next); } ], function (err) { if (err) { console.error( 'Unable to resize ' + srcBucket + '/' + srcKey + ' and upload to ' + dstBucket + '/' + dstKey + ' due to an error: ' + err ); } else { console.log( 'Successfully resized ' + srcBucket + '/' + srcKey + ' and uploaded to ' + dstBucket + '/' + dstKey ); } callback(null, "message"); } ); }; w ߦͷίʔυɺͭͷؔ਺ w ݹ͍ίʔσΟϯάελΠϧ &4 BTZOD  w Ͳ͏΍Βσουίʔυ͕͋Δ w ؍ଌ͢Δҙຯ߹͍ͷগͳ͍໭Γ஋ w ˠࣗಈςετΛॻ͍ͯৼΔ෣͍͕มΘͬͯͳ͍ ͜ͱΛ͔֬Ίͳ͕Βɺ։ൃܧଓʹඋ͑ͯ৽͍͠ ίʔσΟϯάελΠϧ &4 1SPNJTF ʹҠ ߦ͍ͨ͠
  2. // dependencies var async = require('async'); var AWS = require('aws-sdk');

    var gm = require('gm') .subClass({ imageMagick: true }); // Enable ImageMagick integration. var util = require('util'); // constants var MAX_WIDTH = 100; var MAX_HEIGHT = 100; // get reference to S3 client var s3 = new AWS.S3(); exports.handler = function(event, context, callback) { // Read options from the event. console.log("Reading options from event:\n", util.inspect(event, {depth: 5})); var srcBucket = event.Records[0].s3.bucket.name; // Object key may have spaces or unicode non-ASCII characters. var srcKey = decodeURIComponent(event.Records[0].s3.object.key.replace(/\+/g, " ")); var dstBucket = srcBucket + "resized"; var dstKey = "resized-" + srcKey;
  3. var dstBucket = srcBucket + "resized"; var dstKey = "resized-"

    + srcKey; // Sanity check: validate that source and destination are different buckets. if (srcBucket == dstBucket) { callback("Source and destination buckets are the same."); return; } // Infer the image type. var typeMatch = srcKey.match(/\.([^.]*)$/); if (!typeMatch) { callback("Could not determine the image type."); return; } var imageType = typeMatch[1]; if (imageType != "jpg" && imageType != "png") { callback('Unsupported image type: ${imageType}'); return; } // Download the image from S3, transform, and upload to a different S3 bucket. async.waterfall([ function download(next) { // Download the image from S3 into a buffer.
  4. // Download the image from S3, transform, and upload to

    a different S3 bucket. async.waterfall([ function download(next) { // Download the image from S3 into a buffer. s3.getObject({ Bucket: srcBucket, Key: srcKey }, next); }, function transform(response, next) { gm(response.Body).size(function(err, size) { // Infer the scaling factor to avoid stretching the image unnaturally. var scalingFactor = Math.min( MAX_WIDTH / size.width, MAX_HEIGHT / size.height ); var width = scalingFactor * size.width; var height = scalingFactor * size.height; // Transform the image buffer in memory. this.resize(width, height) .toBuffer(imageType, function(err, buffer) { if (err) { next(err);
  5. // Transform the image buffer in memory. this.resize(width, height) .toBuffer(imageType,

    function(err, buffer) { if (err) { next(err); } else { next(null, response.ContentType, buffer); } }); }); }, function upload(contentType, data, next) { // Stream the transformed image to a different S3 bucket. s3.putObject({ Bucket: dstBucket, Key: dstKey, Body: data, ContentType: contentType }, next); } ], function (err) { if (err) { console.error(
  6. Bucket: dstBucket, Key: dstKey, Body: data, ContentType: contentType }, next);

    } ], function (err) { if (err) { console.error( 'Unable to resize ' + srcBucket + '/' + srcKey + ' and upload to ' + dstBucket + '/' + dstKey + ' due to an error: ' + err ); } else { console.log( 'Successfully resized ' + srcBucket + '/' + srcKey + ' and uploaded to ' + dstBucket + '/' + dstKey ); } callback(null, "message"); } ); };
  7. ςετखॱॻWFS w 4όέοτ४උ w TPVSDFόέοτͱEFTUJOBUJPOόέοτΛ࡞੒ w TPVSDFόέοτʹαϯϓϧΦϒδΣΫτΛΞοϓϩʔυ͓ͯ͘͠ w -BNCEBؔ਺Λ࡞੒ͯ͠σϓϩΠ w

    σϓϩΠύοέʔδΛ࡞੒ w OQNJOTUBMMBTZODHN w σΟϨΫτϦؙ͝ͱ;*1ѹॖ w ࣮ߦϩʔϧΛ࡞੒ w *".ίϯιʔϧͰ<3PMF/BNF> <4FMFDU3PMF5ZQF> <"UUBDI1PMJDZ>Λઃఆ w ϩʔϧͷ"3/ΛϝϞ w -BNCEBؔ਺ΛσϓϩΠ w BXTMBNCEBDSFBUFGVODUJPO w ؔ਺ͷ"3/ΛϝϞ w ໨ࢹͰςετ w "NB[PO4αϯϓϧΠϕϯτσʔλΛϑΝΠϧʹอଘ w -BNCEB$-*JOWPLFίϚϯυΛ࣮ߦͯؔ͠਺Λݺͼग़͠ w αϜωΠϧ͕λʔήοτόέοτʹ࡞੒͞Εͨ͜ͱΛ֬ೝ w "84-BNCEBίϯιʔϧͰ-BNCEBؔ਺ͷϩάΛ֬ೝ ॴཁ࣌ؒ෼
  8. AWSTemplateFormatVersion: '2010-09-09' Transform: AWS::Serverless-2016-10-31 Description: Image resize example Resources: SourceBucket:

    Type: AWS::S3::Bucket Properties: BucketName: images DestinationBucket: Type: AWS::S3::Bucket Properties: BucketName: imagesresized ProcessorFunction: Type: AWS::Serverless::Function Properties: Handler: index.handler Runtime: nodejs6.10 CodeUri: ./code Policies: AmazonS3FullAccess Events: ImageUpload: Type: S3 Properties: Bucket: !Ref SourceBucket Events: s3:ObjectCreated:* ςετखॱॻWFS w 4".ఆٛϑΝΠϧ࡞੒ w 4".σϓϩΠ༻4όέοτ४උ w -BNCEBؔ਺Λ࡞੒ͯ͠σϓϩΠ w BXTDMPVEGPSNBUJPOQBDLBHF w BXTDMPVEGPSNBUJPOEFQMPZ w BXTDMJͰςετ w BXTMPHTHFUBXTMBNCEB<MPHJE>ŠXBUDI w BXTTDQIPHFQOHTJNBHFTIPHFQOH w BXTTMTJNBHFTSFTJ[FE ॴཁ࣌ؒ෼
  9. $ SERVICES=s3 make infra . .venv/bin/activate; PYTHONPATH=. exec localstack/mock/infra.py Starting

    local dev environment. CTRL-C to quit. Starting mock S3 (port 4572)... Ready. $ aws --endpoint-url=http://localhost:4572 s3 ls $ aws --endpoint-url=http://localhost:4572 s3 mb s3://my-test-bucket make_bucket: my-test-bucket $ aws --endpoint-url=http://localhost:4572 s3 ls 2006-02-04 01:45:09 my-test-bucket $ aws --endpoint-url=http://localhost:4572 s3 cp ~/hoge.png s3://my-test-bucket upload: hoge.png to s3://my-test-bucket/hoge.png $ aws --endpoint-url=http://localhost:4572 s3 ls s3://my-test-bucket 2017-06-02 11:39:22 7538 hoge.png MPDBMTUBDLΛNBLF΍EPDLFSͰಈ͔͢ endpoint-url を指定すれば aws cli すら欺ける
  10. const assert = require('assert'); const fs = require('fs'); const path

    = require('path'); const AWS = require('aws-sdk'); const s3 = new AWS.S3({ s3ForcePathStyle: true, logger: console, endpoint: new AWS.Endpoint('http://localhost:4572') }); describe('localstack learning', () => { before(() => { return s3.createBucket({Bucket: 'test-bucket'}).promise().then(() => { return s3.putObject({ Bucket: 'test-bucket', Key: 'TQ_LOGO.png', ContentType: 'image/png', Body: fs.readFileSync(path.join(__dirname, '..', 'fixtures', 'TQ_LOGO.png')) }).promise(); }); }); it('s3.getObject', () => { return s3.getObject({Bucket: 'test-bucket', Key: 'TQ_LOGO.png'}).promise().then((res) => { assert(res); }); }); }); MPDBMTUBDLͱ1SPNJTFͷֶशςετΛॻ͍ͯΈΔ 学習テスト: ライブラリ等の 学習のみに特化したテスト
  11. const assert = require('assert'); const fs = require('fs'); const path

    = require('path'); const AWS = require('aws-sdk'); const s3 = new AWS.S3({ s3ForcePathStyle: true, logger: console, endpoint: new AWS.Endpoint('http://localhost:4572') }); describe('localstack learning', () => { before(() => { return s3.createBucket({Bucket: 'test-bucket'}).promise().then(() => { return s3.putObject({ Bucket: 'test-bucket', Key: 'TQ_LOGO.png', ContentType: 'image/png', Body: fs.readFileSync(path.join(__dirname, '..', 'fixtures', 'TQ_LOGO.png')) }).promise(); }); }); it('s3.getObject', () => { return s3.getObject({Bucket: 'test-bucket', Key: 'TQ_LOGO.png'}).promise().then((res) => { assert(res); }); }); }); endpoint を localstack に 向けるだけ! S3 のバケット URL をパス形式に変えておく (ハマりポイント)
  12. const assert = require('assert'); const fs = require('fs'); const path

    = require('path'); const AWS = require('aws-sdk'); const s3 = new AWS.S3({ s3ForcePathStyle: true, logger: console, endpoint: new AWS.Endpoint('http://localhost:4572') }); describe('localstack learning', () => { before(() => { return s3.createBucket({Bucket: 'test-bucket'}).promise().then(() => { return s3.putObject({ Bucket: 'test-bucket', Key: 'TQ_LOGO.png', ContentType: 'image/png', Body: fs.readFileSync(path.join(__dirname, '..', 'fixtures', 'TQ_LOGO.png')) }).promise(); }); }); it('s3.getObject', () => { return s3.getObject({Bucket: 'test-bucket', Key: 'TQ_LOGO.png'}).promise().then((res) => { assert(res); }); }); }); aws-sdk の Promise 機能をモリモリ試す
  13. var MAX_HEIGHT = 100; // get reference to S3 client

    var s3; if (process.env.NODE_ENV === 'production') { s3 = new AWS.S3(); } else { s3 = new AWS.S3({ s3ForcePathStyle: true, logger: console, endpoint: new AWS.Endpoint('http://localhost:4572') }); } exports.handler = function(event, context, callback) { // Read options from the event. Ξϯνύλʔϯ5FTU-PHJDJO1SPEVDUJPO # " %
  14. var AWS = require('aws-sdk'); var s3 = new AWS.S3(); var

    onObjectCreated = require('./on-object-created'); exports.handler = function(event, context, callback) { onObjectCreated({s3, event, callback}); }; ॲཧຊମΛผϞδϡʔϧʹग़͠ɺҾ਺Λ઀߹෦ʹ͢Δ ( 0 0 % // dependencies var async = require('async'); var gm = require('gm') .subClass({ imageMagick: true }); // Enable ImageMagick integration. var util = require('util'); // constants var MAX_WIDTH = 100; var MAX_HEIGHT = 100; module.exports = function({s3, event, callback}) { // Read options from the event. console.log("Reading options from event:\n", util.inspect(event, {depth: 5})); var srcBucket = event.Records[0].s3.bucket.name; ৽نϑΝΠϧPOPCKFDUDSFBUFEKT JOEFYKT
  15. const assert = require('assert'); const AWS = require('aws-sdk'); const s3

    = new AWS.S3({s3ForcePathStyle: true, endpoint: new AWS.Endpoint('http://localhost:4572')}); const onObjectCreated = require('../on-object-created'); describe('localstack based test suite for happy path cases', () => { let now, event; beforeEach(() => { now = new Date().getTime(); event = { Records: [ { s3: { bucket: { name: `test-bucket-${now}` }, object: { key: `TQ_LOGO_${now}.png` } } } ] }; return s3.createBucket({Bucket: `test-bucket-${now}`}).promise() .then(() => s3.createBucket({Bucket: `test-bucket-${now}resized`}).promise()) .then(() => s3.putObject({ Bucket: `test-bucket-${now}`, Key: `TQ_LOGO_${now}.png`, ContentType: 'image/png', Body: fs.readFileSync(path.join(__dirname, '..', 'fixtures', 'TQ_LOGO.png')) }).promise()); }); it('onObjectCreated callback', () => { return new Promise((resolve, reject) => { const callback = (err, message) => err ? reject(err) : resolve(message); onObjectCreated({s3, event, callback}); }).then((message) => { assert(/^message/.test(message)); }); }); ͜ΕͰςετ͕ॻ͚Δ
  16. const assert = require('assert'); const AWS = require('aws-sdk'); const s3

    = new AWS.S3({s3ForcePathStyle: true, endpoint: new AWS.Endpoint('http://localhost:4572')}); const onObjectCreated = require('../on-object-created'); describe('localstack based test suite for happy path cases', () => { let now, event; beforeEach(() => { now = new Date().getTime(); event = { Records: [ { s3: { bucket: { name: `test-bucket-${now}` }, object: { key: `TQ_LOGO_${now}.png` } } } ] }; return s3.createBucket({Bucket: `test-bucket-${now}`}).promise() .then(() => s3.createBucket({Bucket: `test-bucket-${now}resized`}).promise()) .then(() => s3.putObject({ Bucket: `test-bucket-${now}`, Key: `TQ_LOGO_${now}.png`, ContentType: 'image/png', Body: fs.readFileSync(path.join(__dirname, '..', 'fixtures', 'TQ_LOGO.png')) }).promise()); }); it('onObjectCreated callback', () => { return new Promise((resolve, reject) => { const callback = (err, message) => err ? reject(err) : resolve(message); onObjectCreated({s3, event, callback}); }).then((message) => { assert(/^message/.test(message)); }); }); ઀߹෦Λ׆༻ͯ͠ςετΛॻ͘
  17. Body: fs.readFileSync(path.join(__dirname, '..', 'fixtures', 'TQ_LOGO.png')) }).promise()); }); it('onObjectCreated callback', ()

    => { return new Promise((resolve, reject) => { const callback = (err, message) => err ? reject(err) : resolve(message); onObjectCreated({s3, event, callback}); }).then((message) => { assert(/^message/.test(message)); }); }); it('onObjectCreated creates and puts thumbnail into destination bucket', () => { return new Promise((resolve, reject) => { const callback = (err, message) => err ? reject(err) : resolve(message); onObjectCreated({s3, event, callback}); }) .then(() => { return s3.waitFor('objectExists', { Bucket: `test-bucket-${now}resized`, Key: `resized-TQ_LOGO_${now}.png` }).promise(); }) .then((data) => { assert(data); }); }); });
  18. module.exports = function({s3, event, callback}) { // Read options from

    the event. console.log("Reading options from event:\n", util.inspect(event, {depth: 5})); var srcBucket = event.Records[0].s3.bucket.name; // Object key may have spaces or unicode non-ASCII characters. var srcKey = decodeURIComponent(event.Records[0].s3.object.key.replace(/\+/g, " ")); var dstBucket = srcBucket + "resized"; var dstKey = "resized-" + srcKey; // Sanity check: validate that source and destination are different buckets. if (srcBucket == dstBucket) { callback("Source and destination buckets are the same."); return; } // Infer the image type. var typeMatch = srcKey.match(/\.([^.]*)$/); if (!typeMatch) { callback("Could not determine the image type."); return; } var imageType = typeMatch[1]; if (imageType != "jpg" && imageType != "png") { callback('Unsupported image type: ${imageType}'); return; } IUUQTHJUIVCDPNUXBEBDPWFSMBZFM ݱঢ়֬ೝͷͨΊΧόϨοδଌఆ
  19. IUUQTHJUIVCDPNUXBEBDPWFSMBZFM // Transform the image buffer in memory. this.resize(width, height)

    .toBuffer(imageType, function(err, buffer) { if (err) { next(err); } else { next(null, response.ContentType, buffer); } }); }); }, function upload(contentType, data, next) { // Stream the transformed image to a different S3 bucket. s3.putObject({ Bucket: dstBucket, Key: dstKey, Body: data, ContentType: contentType }, next); } ], function (err) { if (err) { console.error( 'Unable to resize ' + srcBucket + '/' + srcKey + ' and upload to ' + dstBucket + '/' + dstKey + ' due to an error: ' + err ); ྫ֎ܥ͕ςετ͞Εͯͳ͍͜ͱ͕Θ͔Δ
  20. const assert = require('assert'); const onObjectCreated = require('../on-object-created'); describe('small sized

    test suite for exceptional cases:', () => { it('file without extension', () => { const event = { Records: [ { s3: { bucket: { name: `test-bucket` }, object: { key: `TQ_LOGO` } } } ] }; onObjectCreated({event, callback: (err, message) => { assert(err === 'Could not determine the image type.'); }}); }); it('extension is not `.png` nor `.jpg`', () => { const event = { Records: [ { s3: { bucket: { name: `test-bucket` }, object: { key: `TQ_LOGO.txt` } } } ] }; onObjectCreated({event, callback: (err, message) => { assert(err === 'Unsupported image type: txt'); }}); }); }); MPDBMTUBDL͢Β࢖Θͳ͍TNBMMUFTUͰྫ֎ܥΛςετ͢Δ
  21. IUUQTHJUIVCDPNQPXFSBTTFSUKTQPXFSBTTFSU > mocha --require intelli-espower-loader 'test/small/**/*.js' small sized test suite

    for exceptional cases: ✓ file without extension 1) extension is not `.png` nor `.jpg` 1 passing (47ms) 1 failing 1) small sized test suite for exceptional cases: extension is not `.png` nor `.jpg`: AssertionError: # test/small/exceptional_cases_test.js:25 assert(err === 'Unsupported image type: txt') | | | false "Unsupported image type: ${imageType}" --- [string] 'Unsupported image type: txt' +++ [string] err @@ -21,7 +21,16 @@ pe: -txt +${imageType} _人人人人人人人人人人_ > 突然のテスト失敗 <  ̄YYYYYYYYYY ̄
  22. IUUQTHJUIVCDPNQPXFSBTTFSUKTQPXFSBTTFSU > mocha --require intelli-espower-loader 'test/small/**/*.js' small sized test suite

    for exceptional cases: ✓ file without extension 1) extension is not `.png` nor `.jpg` 1 passing (47ms) 1 failing 1) small sized test suite for exceptional cases: extension is not `.png` nor `.jpg`: AssertionError: # test/small/exceptional_cases_test.js:25 assert(err === 'Unsupported image type: txt') | | | false "Unsupported image type: ${imageType}" --- [string] 'Unsupported image type: txt' +++ [string] err @@ -21,7 +21,16 @@ pe: -txt +${imageType} ڭ܇ςετͯ͠ͳ͍ίʔυ͸ಈ͔ͳ͍ if (imageType != "jpg" && imageType != "png") { callback('Unsupported image type: ${imageType}'); return; }
  23. AWSTemplateFormatVersion: '2010-09-09' Transform: AWS::Serverless-2016-10-31 Description: Image resize example Resources: SourceBucket:

    Type: AWS::S3::Bucket Properties: BucketName: test-{{branchName}} DestinationBucket: Type: AWS::S3::Bucket Properties: BucketName: test-{{branchName}}resized ProcessorFunction: Type: AWS::Serverless::Function Properties: Handler: index.handler Runtime: nodejs6.10 CodeUri: ./code Policies: AmazonS3FullAccess Events: ImageUpload: Type: S3 Properties: Bucket: !Ref SourceBucket Events: s3:ObjectCreated:* ϒϥϯνຖʹҟͳΔ؀ڥΛ࡞Δ 手に馴染んだテンプレートエンジンを使って SAM 定義をブランチ毎に生成する
  24. const assert = require('assert'); const fs = require('fs'); const path

    = require('path'); const AWS = require('aws-sdk'); const s3 = new AWS.S3({ logger: console }); const execSync = require('child_process').execSync; const branch = execSync('git rev-parse --abbrev-ref @').toString().replace(/\n$/, ''); describe('large sized test suite', () => { it('happy path', () => { const now = new Date().getTime(); return Promise.resolve() .then(() => { return s3.putObject({ Bucket: `test-${branch}`, Key: `TQ_LOGO_${now}.png`, Body: fs.readFileSync(path.join(__dirname, '..', 'fixtures', 'TQ_LOGO.png')), ContentType: 'image/png' }).promise(); }) .then(() => { return s3.waitFor('objectExists', { Bucket: `test-${branch}resized`, Key: `resized-TQ_LOGO_${now}.png` }).promise(); }) .then((data) => { assert(data); }); }); }); MBSHFϒϥϯνผͷ"84؀ڥʹର͢Δςετ
  25. const gm = require('gm').subClass({ imageMagick: true }); const util =

    require('util'); const resize = require('./resize'); module.exports = function onObjectCreated ({s3, event, callback}) { console.log('Reading options from event:\n', util.inspect(event, {depth: 5})); const srcBucket = event.Records[0].s3.bucket.name; const srcKey = decodeURIComponent(event.Records[0].s3.object.key.replace(/\+/g, ' ')); const dstBucket = srcBucket + 'resized'; const dstKey = 'resized-' + srcKey; resize({s3, gm, srcBucket, srcKey, dstBucket, dstKey}) .then((message) => { console.log(message); callback(null, message); }) .catch((err) => { const message = `Unable to resize ${srcBucket}/${srcKey} and upload to ${dstBucket}/ ${dstKey} due to an error: ${err}`; console.error(message); callback(null, message); }); }; "84-BNCEBʹґଘͨ͠ཁૉΛઙ͍֊૚ͰҾ͖ണ͕͢