diff --git a/README.md b/README.md index 48eefb7..4493b68 100644 --- a/README.md +++ b/README.md @@ -1,36 +1,49 @@ # s3-deploy -A command line tool to deploy static assets to an S3 bucket. +A command line tool to deploy static sites to an S3 bucket. + +## Why? + +At [Moneylion](https://moneylion.com), we have a lot of web properties. In order to untangle some of our deploy processes for frontend assets, we developed this script which allows us to quickly and painlessly configure a new deployment to S3. + +One of the main uses for this script is to create temporary review-apps. When a new PR is created, this triggers a build in our CI pipeline, if all the tests pass and it builds successfully, then we deploy that PR to a temporary and shareable URL. Once the PR is closed, we tear it down. ## Required ENV Variables -- NPM_TOKEN - AWS_SECRET_ACCESS_KEY - AWS_ACCESS_KEY_ID - AWS_DEFAULT_REGION ### optional -- CODEFRESH_SLACK_BOT_TOKEN +- SLACK_TOKEN ## Usage `npx @moneylion/s3-deploy` +## Arguments + +| argument | description | +| -------------- | ------------------------------ | +| `domain` | A fully qualified domain name. | +| `zone` | The route53 HostedZoneId. | +| `distribution` | The CloudFront DistributionId. | +| `channel` | The slack channel name. | + ## Commands ### `deploy` -This will create a new S3 bucket and point route53 at it. The last argument must be the directoy to be uploaded. +This will create a new S3 bucket and point route53 at it. The last argument must be the directory to be uploaded. If an S3 bucket with this name already exists, then it will be cleared before the new files are uploaded. requires: -- host -- bucket +- domain - zone e.g.
-`npx @moneylion/s3-deploy deploy --host example.com --bucket test --zone Z2XDC2IJ26IK32 ./dist` +`npx @moneylion/s3-deploy deploy --domain test.example.com --zone Z2XDC2IJ26IK32 ./dist` ### `undeploy` @@ -38,37 +51,32 @@ This will delete an S3 bucket and the route53 record set. requires: -- host -- bucket +- domain - zone +optional + +- channel + e.g.
-`npx @moneylion/s3-deploy undeploy --host example.com --bucket test --zone Z2XDC2IJ26IK32` +`npx @moneylion/s3-deploy undeploy --domain example.com --zone Z2XDC2IJ26IK32` ### `promote` +_Creating a cloudfront distribution is outside the scope of this script. There are no future plans to support this feature._ + > requires an already existing bucket
> requires an already existing cloudfront distribution -This will not create a new bucket, it will instead empty it and upload the new assets to that bucket. It will then invalidate the cloudfront cache. +This will not create a new bucket, it will instead empty an already existing and then upload the new assets to that bucket. It will also invalidate the cloudfront cache. requires: -- host -- bucket +- domain - distribution e.g.
-`npx @moneylion/s3-deploy promote --host example.com --bucket test --distribution EINBTGEF4J77C ./dist` - ---- - -Bucket names are constructed by concatenating the `host` and `bucket` strings. The last argument must be the directoy to be uploaded. - -bucket = test
-host = example.com - -Will give you `test.example.com` as the bucket name. +`npx @moneylion/s3-deploy promote --domain test.example.com --distribution ABCDEFGH1I23J ./dist` --- @@ -78,18 +86,6 @@ Both `deploy` and `promote` can take in a `channel` argument, this will be the s ## Troubleshooting -### Getting 404's from npm - -This is a private script and it reads from `.npmrc` for the auth token. - -A quick & dirty workaround for CI or Docker is to create a local `.npmrc` before running the script. - -`echo "//registry.npmjs.org/:_authToken=${NPM_TOKEN}" > $(pwd)/.npmrc` - -Then run it with the `--userconfig` flag. - -`npx --userconfig $(pwd)/.npmrc @moneylion/s3-deploy` - ### Undefined environment variables It might be necessary to inject the environment variable directly into the script. diff --git a/jest.config.js b/jest.config.js index 15dc658..3fc8d07 100644 --- a/jest.config.js +++ b/jest.config.js @@ -1,21 +1,17 @@ module.exports = { verbose: true, transform: { - "^.+\\.ts$": "ts-jest" - }, - testRegex: "(/__tests__/.*|(\\.|/)(test|spec))\\.(js|ts)$", - moduleDirectories: ["node_modules", "app"], - moduleFileExtensions: ["ts", "js", "json", "node"], - moduleNameMapper: { - "\\.(css|eot|woff|woff2)$": "/app/spec/__mocks__/styleMock.js", - "util/jss": "/app/spec/__mocks__/jssMock.js" + '^.+\\.ts$': 'ts-jest' }, + testRegex: 'test\\.ts$', + moduleDirectories: ['node_modules', 'app'], + moduleFileExtensions: ['ts', 'js', 'json', 'node'], globals: { - "ts-jest": { - tsConfigFile: "tsconfig.json" + 'ts-jest': { + tsConfigFile: 'tsconfig.json' } }, - roots: ["/app"], + roots: ['/src'], collectCoverage: true, - coveragePathIgnorePatterns: ["/node_modules"] + coveragePathIgnorePatterns: ['/node_modules'] } diff --git a/package.json b/package.json index fc9f4eb..d3d2063 100644 --- a/package.json +++ b/package.json @@ -1,12 +1,12 @@ { "name": "@moneylion/s3-deploy", - "version": "1.3.1", + "version": "2.0.0", "engines": { "node": ">=11.13.0", "yarn": ">=1.15.0" }, - "main": "./dist/ops.js", - "bin": "./dist/ops.js", + "main": "./dist/index.js", + "bin": "./dist/index.js", "scripts": { "lint": "tslint --project ./tsconfig.json", "lint:fix": "tslint --fix --project ./tsconfig.json", diff --git a/src/cli.ts b/src/cli.ts new file mode 100644 index 0000000..71cb5e7 --- /dev/null +++ b/src/cli.ts @@ -0,0 +1,129 @@ +#!/usr/bin/env node + +import mri from 'mri' + +interface ParsedArgs { + action: 'deploy' | 'undeploy' | 'promote' + domain: string + + zone?: string + distribution?: string + channel?: string + dir?: string +} + +const errors = { + domain: { + req: 'Domain is required', + char: "Domain can only be a-z, 0-9, '.', and '-'." + }, + zone: { + req: 'Zone is required', + char: 'Zone can only be A-Z and 0-9.', + len: 'Zone has to be 14 characters long.' + }, + directory: { req: 'Directory is required' }, + distribution: { req: 'Distribution is required' } +} + +const parseArgs = (argv: string[]): ParsedArgs => { + const args = mri(argv, { string: ['domain', 'zone', 'channel'] }) + + const { + _: [action, dir], + domain, + zone, + distribution, + channel + } = args + + return { action, dir, domain, zone, channel, distribution } as ParsedArgs +} + +const forcedExit = (msg: string) => { + console.error(msg) + process.exit(1) +} + +const isValidAction = (action: string) => { + const allowedActions = new Set(['promote', 'deploy', 'undeploy']) + + if (!allowedActions.has(action)) + return `${action} is not a valid action. Allowed actions: ${Array.from(allowedActions.values()).join(', ')}` + + return true +} + +const isValidDomain = (domain: string) => { + if (typeof domain === 'undefined') return errors.domain.req + if (!/^[a-z0-9-\.]+$/.test(domain)) return errors.domain.char + + return true +} + +const isValidZone = (zone: string) => { + if (typeof zone === 'undefined') return errors.zone.req + if (!/^[A-Z0-9]+$/.test(zone)) return errors.zone.char + if (zone.length !== 14) return errors.zone.len + + return true +} + +const isValidDir = (dir: string) => { + if (typeof dir === 'undefined') return errors.directory.req + + return true +} + +const isValidDistribution = (distribution: string) => { + if (typeof distribution === 'undefined') return errors.distribution.req + + return true +} + +const deploy = ({ domain, zone, dir, channel }: ParsedArgs) => { + const validDomain = isValidDomain(domain) + const validZone = isValidZone(zone) + const validDir = isValidDir(dir) + + if (validDomain !== true) return forcedExit(validDomain) + if (validZone !== true) return forcedExit(validZone) + if (validDir !== true) return forcedExit(validDir) + + require('./deploy').deploy(domain, zone, dir, channel) +} + +const undeploy = ({ domain, zone }: ParsedArgs) => { + const validDomain = isValidDomain(domain) + const validZone = isValidZone(zone) + + if (validDomain !== true) return forcedExit(validDomain) + if (validZone !== true) return forcedExit(validZone) + + require('./undeploy').undeploy(domain, zone) +} + +const promote = ({ domain, distribution, dir, channel }: ParsedArgs) => { + const validDomain = isValidDomain(domain) + const validDistribution = isValidDistribution(distribution) + const validDir = isValidDir(dir) + + if (validDomain !== true) return forcedExit(validDomain) + if (validDistribution !== true) return forcedExit(validDistribution) + if (validDir !== true) return forcedExit(validDir) + + require('./promote').promote(domain, distribution, dir, channel) +} + +export const cli = (argv: string[]) => { + const args = parseArgs(argv) + + const { action } = args + + const validAction = isValidAction(action) + if (validAction !== true) return forcedExit(validAction) + + if (action === 'deploy') deploy(args) + else if (action === 'undeploy') undeploy(args) + else if (action === 'promote') promote(args) +} diff --git a/src/deploy.test.ts b/src/deploy.test.ts new file mode 100644 index 0000000..7477940 --- /dev/null +++ b/src/deploy.test.ts @@ -0,0 +1,102 @@ +console.log = jest.fn() + +jest.mock('fs') +jest.mock('mime-types') + +const mockSync = jest.fn() +jest.mock('fast-glob', () => ({ sync: mockSync })) + +const mockCreateBucket = jest.fn() +const mockSetPolicy = jest.fn() +const mockSetWebsite = jest.fn() +const mockUpload = jest.fn() +const mockCheckBucket = jest.fn() +const mockEmptyBucket = jest.fn() + +jest.mock('./aws/s3', () => ({ + createBucket: mockCreateBucket, + setPolicy: mockSetPolicy, + setWebsite: mockSetWebsite, + upload: mockUpload, + checkBucket: mockCheckBucket, + emptyBucket: mockEmptyBucket +})) + +const mockCreateRecordSet = jest.fn() + +jest.mock('./aws/route53', () => ({ createRecordSet: mockCreateRecordSet })) + +const mockPostToChannel = jest.fn() + +jest.mock('./slack/message', () => ({ postToChannel: mockPostToChannel })) + +import { deploy } from './deploy' + +describe('deploy', () => { + const domain = 'test.example.com' + const zone = 'ABC123' + const dir = '.' + const files = ['test', 'file', 'foo'] + + afterEach(() => jest.clearAllMocks()) + + test('creates a bucket if none exists', async () => { + mockSync.mockReturnValueOnce(files) + + mockCheckBucket.mockResolvedValueOnce(false) + + await deploy(domain, zone, dir) + + expect(mockCreateBucket).toHaveBeenCalledTimes(1) + expect(mockEmptyBucket).toHaveBeenCalledTimes(0) + }) + + test('creates a record set if no bucket exists', async () => { + mockSync.mockReturnValueOnce(files) + + mockCheckBucket.mockResolvedValueOnce(false) + + await deploy(domain, zone, dir) + + expect(mockCreateRecordSet).toHaveBeenCalledTimes(1) + expect(mockCreateRecordSet).toHaveBeenCalledWith(domain, zone) + }) + + test('will throw error if no files found', () => { + mockSync.mockReturnValueOnce([]) + + expect(deploy(domain, zone, dir)).rejects.toThrowError() + }) + + test("doesn't creates a bucket if one exists", async () => { + mockSync.mockReturnValueOnce(files) + + mockCheckBucket.mockResolvedValueOnce(true) + + await deploy(domain, zone, dir) + + expect(mockCreateBucket).toHaveBeenCalledTimes(0) + expect(mockEmptyBucket).toHaveBeenCalledTimes(1) + }) + + test('uploads all found files to bucket', async () => { + mockSync.mockReturnValueOnce(files) + + mockCheckBucket.mockResolvedValueOnce(true) + + await deploy(domain, zone, dir) + + expect(mockUpload).toHaveBeenCalledTimes(files.length) + }) + + test('will post to slack channel if provided', async () => { + const channel = 'channel' + mockSync.mockReturnValueOnce(files) + + mockCheckBucket.mockResolvedValueOnce(false) + + await deploy(domain, zone, dir, channel) + + expect(mockPostToChannel).toHaveBeenCalledTimes(1) + }) +}) diff --git a/src/deploy.ts b/src/deploy.ts index 31a1260..a0648ef 100755 --- a/src/deploy.ts +++ b/src/deploy.ts @@ -17,18 +17,19 @@ interface FileType { contentType: string } -export const deploy = async (bucket: string, host: string, zone: string, dir: string, channel: string) => { +export const deploy = async (domain: string, zone: string, dir: string, channel?: string) => { const files = getFiles(dir) - const fqdn = `${bucket}.${host}` - const bucketExists = await checkBucket(fqdn) + const bucketExists = await checkBucket(domain) - if (!bucketExists) await makeBucket(fqdn) - else await emptyBucket(fqdn) + if (!bucketExists) await makeBucket(domain) + else await emptyBucket(domain) - await uploadFiles(fqdn, files) + await uploadFiles(domain, files) - if (!bucketExists) await makeRecordSet(fqdn, zone, channel) + if (!bucketExists) await makeRecordSet(domain, zone) + + if (typeof channel !== 'undefined') await postToChannel(channel, `Deployed app: ${domain}`) console.log('Done.') } @@ -47,32 +48,30 @@ const getFiles = (dir: string): FileType[] => { return files } -const makeBucket = async (fqdn: string) => { - await createBucket(fqdn) +const makeBucket = async (domain: string) => { + await createBucket(domain) console.log('Done bucket.') - await setPolicy(fqdn) + await setPolicy(domain) console.log('Done policy.') - await setWebsite(fqdn) + await setWebsite(domain) console.log('Done website.') } -const uploadFiles = async (fqdn: string, files: FileType[]) => { +const uploadFiles = async (domain: string, files: FileType[]) => { const total = files.length let done = 0 const promises = files.map(async ({ key, body, contentType }) => { - await upload(fqdn, key, body, contentType) + await upload(domain, key, body, contentType) console.log(`Done ${++done}/${total}.`) }) await Promise.all(promises) } -const makeRecordSet = async (fqdn: string, zone: string, channel: string) => { - await createRecordSet(fqdn, zone) +const makeRecordSet = async (domain: string, zone: string) => { + await createRecordSet(domain, zone) console.log('Done route53.') - const text = `Deployed app: ${fqdn}` - if (typeof channel !== 'undefined') await postToChannel(channel, text) } diff --git a/src/index.ts b/src/index.ts new file mode 100644 index 0000000..0e56a28 --- /dev/null +++ b/src/index.ts @@ -0,0 +1,3 @@ +import { cli } from './cli' + +cli(process.argv.slice(2)) diff --git a/src/ops.ts b/src/ops.ts deleted file mode 100644 index e3818c1..0000000 --- a/src/ops.ts +++ /dev/null @@ -1,50 +0,0 @@ -#!/usr/bin/env node - -import mri from 'mri' - -const argv = process.argv.slice(2) -const args = mri(argv, { string: ['bucket', 'host', 'zone', 'channel'] }) - -const { - _: [action, dir], - bucket, - host, - zone, - channel, - distribution -} = args - -const allowedActions = new Set(['promote', 'deploy', 'undeploy']) - -if (!allowedActions.has(action)) { - console.log(`${action} is not a valid action. Allowed actions: ${Array.from(allowedActions.values()).join(', ')}`) - process.exit(1) -} - -if (action === 'deploy' || action === 'undeploy') { - if (typeof zone === 'undefined') throw new Error('Zone is required') - if (!/^[A-Z0-9]+$/.test(zone)) throw new Error(`Zone can only be A-Z and 0-9: ${zone}`) - if (zone.length !== 14) throw new Error(`Zone has to be 14 characters long: ${zone}`) -} - -if (action === 'deploy' || action === 'promote') { - if (typeof dir === 'undefined') throw new Error('Directory is required') -} - -if (typeof host === 'undefined') throw new Error('Host is required') -if (typeof bucket === 'undefined') throw new Error('Bucket is required') - -if (!/^[a-z0-9-\.]+$/.test(host)) throw new Error(`Host can only be a-z, 0-9, '.', and '-': ${host}`) -if (!/^[a-z0-9-]+$/.test(bucket)) throw new Error(`Bucket can only be a-z, 0-9 and '-': ${bucket}`) - -if (action === 'deploy') { - // tslint:disable-next-line:no-var-requires - require('./deploy').deploy(bucket, host, zone, dir, channel) -} else if (action === 'undeploy') { - // tslint:disable-next-line:no-var-requires - require('./undeploy').undeploy(bucket, host, zone) -} else if (action === 'promote') { - if (typeof distribution === 'undefined') throw new Error('Distribution is required') - // tslint:disable-next-line:no-var-requires - require('./promote').promote(bucket, host, distribution, dir, channel) -} diff --git a/src/promote.test.ts b/src/promote.test.ts new file mode 100644 index 0000000..1521e6a --- /dev/null +++ b/src/promote.test.ts @@ -0,0 +1,74 @@ +console.log = jest.fn() + +jest.mock('fs') +jest.mock('mime-types') + +const mockSync = jest.fn() +jest.mock('fast-glob', () => ({ sync: mockSync })) + +const mockUpload = jest.fn() +const mockEmptyBucket = jest.fn() + +jest.mock('./aws/s3', () => ({ + upload: mockUpload, + emptyBucket: mockEmptyBucket +})) + +const mockCreateInvalidation = jest.fn() + +jest.mock('./aws/cloudfront', () => ({ createInvalidation: mockCreateInvalidation })) + +const mockPostToChannel = jest.fn() + +jest.mock('./slack/message', () => ({ postToChannel: mockPostToChannel })) + +import { promote } from './promote' + +describe('promote', () => { + const domain = 'test.example.com' + const distribution = 'ABC123' + const dir = '.' + const files = ['test', 'file', 'foo'] + + afterEach(() => jest.clearAllMocks()) + + test('will throw error if no files found', () => { + mockSync.mockReturnValueOnce([]) + + expect(promote(domain, distribution, dir)).rejects.toThrowError() + }) + + test('empties the bucket before uploading', async () => { + mockSync.mockReturnValueOnce(files) + + await promote(domain, distribution, dir) + + expect(mockEmptyBucket).toHaveBeenCalledTimes(1) + expect(mockEmptyBucket).toHaveBeenCalledWith(domain) + }) + + test('uploads the files', async () => { + mockSync.mockReturnValueOnce(files) + + await promote(domain, distribution, dir) + + expect(mockUpload).toHaveBeenCalledTimes(files.length) + }) + + test('creates cloudfront invalidation', async () => { + mockSync.mockReturnValueOnce(files) + + await promote(domain, distribution, dir) + + expect(mockCreateInvalidation).toHaveBeenCalledTimes(1) + }) + + test('will post to slack channel if provided', async () => { + const channel = 'channel' + mockSync.mockReturnValueOnce(files) + + await promote(domain, distribution, dir, channel) + + expect(mockPostToChannel).toHaveBeenCalledTimes(1) + }) +}) diff --git a/src/promote.ts b/src/promote.ts index ab19734..9ef54d9 100755 --- a/src/promote.ts +++ b/src/promote.ts @@ -15,21 +15,14 @@ interface FileType { contentType: string } -export const promote = async ( - bucket: string, - host: string, - distribution: string, - dir: string, - channel: string = null -) => { +export const promote = async (domain: string, distribution: string, dir: string, channel: string = null) => { const files = getFiles(dir) - const fqdn = `${bucket}.${host}` - await emptyBucket(fqdn) - await uploadFiles(fqdn, files) + await emptyBucket(domain) + await uploadFiles(domain, files) await createInvalidation(distribution) - const text = `Deployed ${fqdn}` + const text = `Deployed ${domain}` if (channel) await postToChannel(channel, text) console.log('Done.') @@ -49,12 +42,12 @@ const getFiles = (dir: string): FileType[] => { return files } -const uploadFiles = async (fqdn: string, files: FileType[]) => { +const uploadFiles = async (domain: string, files: FileType[]) => { const total = files.length let done = 0 const promises = files.map(async ({ key, body, contentType }) => { - await upload(fqdn, key, body, contentType) + await upload(domain, key, body, contentType) console.log(`Done ${++done}/${total}.`) }) diff --git a/src/slack/message.ts b/src/slack/message.ts index 41bf1d9..ab7e9bc 100644 --- a/src/slack/message.ts +++ b/src/slack/message.ts @@ -4,7 +4,7 @@ dotenv.config() import request from 'request' const API_BASE = 'https://slack.com/api' -const token = process.env.CODEFRESH_SLACK_BOT_TOKEN +const token = process.env.SLACK_TOKEN export const postToChannel = (channel: string, text: string) => { const url = `${API_BASE}/chat.postMessage` diff --git a/src/undeploy.test.ts b/src/undeploy.test.ts new file mode 100644 index 0000000..8777d59 --- /dev/null +++ b/src/undeploy.test.ts @@ -0,0 +1,25 @@ +const mockDeleteBucket = jest.fn() +const mockDeleteRecordSet = jest.fn() + +jest.mock('./aws/s3', () => ({ deleteBucket: mockDeleteBucket })) +jest.mock('./aws/route53', () => ({ deleteRecordSet: mockDeleteRecordSet })) + +import { undeploy } from './undeploy' + +describe('undeploy', () => { + test('can undeploy', async () => { + mockDeleteBucket.mockResolvedValueOnce(true) + mockDeleteRecordSet.mockResolvedValueOnce(true) + + const domain = 'test.example.com' + const zone = 'ABC123' + + await undeploy(domain, zone) + + expect(mockDeleteBucket).toHaveBeenCalledTimes(1) + expect(mockDeleteBucket).toHaveBeenCalledWith(domain) + + expect(mockDeleteRecordSet).toHaveBeenCalledTimes(1) + expect(mockDeleteRecordSet).toHaveBeenCalledWith(domain, zone) + }) +}) diff --git a/src/undeploy.ts b/src/undeploy.ts index 5b6244d..4ecf16b 100755 --- a/src/undeploy.ts +++ b/src/undeploy.ts @@ -1,8 +1,7 @@ import { deleteBucket } from './aws/s3' import { deleteRecordSet } from './aws/route53' -export const undeploy = async (bucket: string, host: string, zone: string) => { - const fqdn = `${bucket}.${host}` - await deleteBucket(fqdn) - await deleteRecordSet(fqdn, zone) +export const undeploy = async (domain: string, zone: string) => { + await deleteBucket(domain) + await deleteRecordSet(domain, zone) } diff --git a/tsconfig.json b/tsconfig.json index 8e1f703..124a784 100644 --- a/tsconfig.json +++ b/tsconfig.json @@ -7,5 +7,6 @@ "esModuleInterop": true, "allowSyntheticDefaultImports": true, "outDir": "./dist" - } + }, + "exclude": ["./**/*.test.ts"] }