Skip to content
This repository has been archived by the owner on Apr 10, 2020. It is now read-only.

Commit

Permalink
add scp deploy mode
Browse files Browse the repository at this point in the history
  • Loading branch information
@mbjorkegren
mbjorkegren committed Mar 8, 2018
1 parent ddd0638 commit 64056d0
Show file tree
Hide file tree
Showing 4 changed files with 207 additions and 55 deletions.
2 changes: 2 additions & 0 deletions .gitignore
View file
Original file line number Diff line number Diff line change
Expand Up @@ -7,3 +7,5 @@ dist
node_modules
npm-debug.log
.polis_s3_creds_client.json
.polis_scp_creds_client.json
polis.config.js
243 changes: 188 additions & 55 deletions gulpfile.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@

var _ = require('lodash');
var exec = require('child_process').exec;
var glob = require('glob');
var gulp = require('gulp');
var s3 = require('gulp-s3');
var rename = require('gulp-rename');
Expand All @@ -11,6 +12,11 @@ var Promise = require('es6-promise').Promise;
var fs = require('fs');
var rimraf = require("rimraf");
var runSequence = require('run-sequence');
var scp = require('gulp-scp2');

var polisConfig = require('./polis.config');

console.log("Uploader: " + polisConfig.UPLOADER);

const staticFilesPrefix = "cached";
const baseDistRoot = "dist";
Expand Down Expand Up @@ -84,10 +90,14 @@ gulp.task('index', [

gulp.task("preprodConfig", function() {
preprodMode = true;
scpSubdir = polisConfig.SCP_SUBDIR_PREPROD;
s3Subdir = polisConfig.S3_BUCKET_PREPROD;
});

gulp.task("prodConfig", function() {
prodMode = true;
scpSubdir = polisConfig.SCP_SUBDIR_PROD;
s3Subdir = polisConfig.S3_BUCKET_PROD;
});


Expand Down Expand Up @@ -151,23 +161,195 @@ gulp.task('dist', [
});


function s3uploader(params) {
var creds = JSON.parse(fs.readFileSync('.polis_s3_creds_client.json'));
creds = _.extend(creds, params);
let f = function(o) {
let oo = _.extend({
delay: 1000,
makeUploadPath: function(file) {
let r = staticFilesPrefix + ".*";
let match = file.path.match(RegExp(r));
console.log(file);
console.log(file.path);
console.log(r, match);

let fixed = (_.isString(o.subdir) && match && match[0]) ? match[0] : path.basename(file.path);
console.log("upload path " + fixed);
return fixed;
},
}, o);
if (oo.headers) {
delete oo.headers['Content-Type']; // s3 figures this out
}

return s3(creds, oo);
};
f.needsHeadersJson = false;
return f;
}

function scpUploader(params) {
var creds = JSON.parse(fs.readFileSync('.polis_scp_creds_client.json'));
creds.dest = path.join(creds.dest, scpSubdir);
scpConfig = _.extend({}, creds, params);
let f = function(batchConfig) { // uploader
console.log("scpUploader run", batchConfig);
var o = _.extend({}, scpConfig);
if (batchConfig.subdir) {
console.log("batchConfig.subdir", batchConfig.subdir);
console.log('old path', o.dest);
o.dest = path.join(scpConfig.dest, batchConfig.subdir);
console.log('new path', o.dest);
} else {
console.log('basic path', o.dest);
}
// console.log('------------------------ foofoo', batchConfig);
// return foreach(function(stream, file){
// console.log('------------------------ stringSrc', file);
// return mergeStream(stream, stringSrc(file.name + ".headersJson", JSON.stringify(batchConfig.headers)));
// }).pipe(scp(scpConfig));
return scp(o);
};
f.needsHeadersJson = true;
return f;
}


function deploy(uploader) {

var cacheSecondsForContentWithCacheBuster = 31536000;

function makeUploadPathHtml(file) {
var fixed = file.path.match(RegExp("[^/]*$"))[0];
console.log("upload path: " + fixed);
return fixed;
}

function makeUploadPathFactory(tagForLogging) {
return function(file) {
console.log(file);
console.log(file.path);
let r = staticFilesPrefix + ".*";
let match = file.path.match(RegExp(r));

let fixed = match&&match[0] ? match[0] : file.path;
console.log("upload path " + tagForLogging + ": " + fixed);
return fixed;
}
}

function deployBatch({srcKeep, srcIgnore, headers, logStatement, subdir}) {
return new Promise(function(resolve, reject) {
let gulpSrc = [srcKeep];
if (srcIgnore) {
gulpSrc.push(srcIgnore);
}
function doDeployBatch() {
console.log("doDeployBatch", gulpSrc);

gulp.src(gulpSrc, {read: true})
.pipe(uploader({
subdir: subdir,
delay: 1000,
headers: headers,
})).on('error', function(err) {
console.log('error1', err);
reject(err);
}).on('end', resolve);
}
// create .headersJson files
if (uploader.needsHeadersJson) {
let globOpts = {
nodir: true,
};
if (srcIgnore) {
globOpts.ignore = srcIgnore;
}
let files = glob.sync(srcKeep, globOpts)
for (let i = 0; i < files.length; i++) {
let file = files[i];
let headerFilename = file + ".headersJson";
fs.writeFileSync(headerFilename, JSON.stringify(headers));
gulpSrc.push(headerFilename);
}
}
doDeployBatch();
});
}

const promises = [];
// Cached Files without Gzip
console.log(destRoot())

const cachedSubdir = "cached";

// Cached Gzipped JS Files
promises.push(deployBatch({
srcKeep: destRoot() + '**/js/**', // simply saying "/js/**" causes the 'js' prefix to be stripped, and the files end up in the root of the bucket.
headers: {
'x-amz-acl': 'public-read',
'Content-Encoding': 'gzip',
'Content-Type': 'application/javascript',
'Cache-Control': 'no-transform,public,max-age=MAX_AGE,s-maxage=MAX_AGE'.replace(/MAX_AGE/g, cacheSecondsForContentWithCacheBuster),
},
logStatement: makeUploadPathFactory("cached_gzipped_"+cacheSecondsForContentWithCacheBuster),
subdir: cachedSubdir,
}));


// HTML files (uncached)
// (Wait until last to upload the html, since it will clobber the old html on S3, and we don't want that to happen before the new JS/CSS is uploaded.)
promises.push(deployBatch({
srcKeep: destRootBase + '/**/*.html',
headers: {
'x-amz-acl': 'public-read',
'Content-Type': 'text/html; charset=UTF-8',
'Cache-Control': 'no-cache',
// 'Cache-Control': 'no-transform,public,max-age=0,s-maxage=300', // NOTE: s-maxage is small for now, we could bump this up later once confident in cloudflare's cache purge workflow
},
logStatement: makeUploadPathHtml,
subdir: null,
}));

return Promise.all(promises);
}

function doUpload() {
var uploader;
if ('s3' === polisConfig.UPLOADER) {
uploader = s3uploader({
bucket: s3Subdir,
});
}
if ('scp' === polisConfig.UPLOADER) {
uploader = scpUploader({
// subdir: "cached",
watch: function(client) {
client.on('write', function(o) {
console.log('write %s', o.destination);
});
},
});
}
return deploy(uploader);
}

gulp.task('deploy_TO_PRODUCTION', [
"prodConfig",
"dist"
], function() {

return deploy({
bucket: "pol.is"
doUpload().catch((err) => {
console.error(err);
});
});

gulp.task('deployPreprod', [
"preprodConfig",
"dist"
], function() {

return deploy({
bucket: "preprod.pol.is"
doUpload().catch((err) => {
console.error(err);
});
});

Expand Down Expand Up @@ -228,55 +410,6 @@ function deployFonts(params) {

}

function deploy(params) {
var creds = JSON.parse(fs.readFileSync('.polis_s3_creds_client.json'));
creds = _.extend(creds, params);

var cacheSecondsForContentWithCacheBuster = 31536000;

function makeUploadPathHtml(file) {
var fixed = file.path.match(RegExp("[^/]*$"))[0];
console.log("upload path: " + fixed);
return fixed;
}

function makeUploadPathFactory(tagForLogging) {
return function(file) {
var fixed = file.path.match(RegExp(staticFilesPrefix + ".*"))[0];
console.log("upload path " + tagForLogging + ": " + fixed);
return fixed;
}
}

// Cached Gzipped Files
gulp.src([
destRoot() + '**/js/**', // simply saying "/js/**" causes the 'js' prefix to be stripped, and the files end up in the root of the bucket.
], {read: false})
.pipe(s3(creds, {
delay: 1000,
headers: {
'x-amz-acl': 'public-read',
'Content-Encoding': 'gzip',
'Cache-Control': 'no-transform,public,max-age=MAX_AGE,s-maxage=MAX_AGE'.replace(/MAX_AGE/g, cacheSecondsForContentWithCacheBuster),
},
makeUploadPath: makeUploadPathFactory("cached_gzipped_"+cacheSecondsForContentWithCacheBuster),
}));

// HTML files (uncached)
// (Wait until last to upload the html, since it will clobber the old html on S3, and we don't want that to happen before the new JS/CSS is uploaded.)
gulp.src([
destRootBase + '/**/*.html',
], {read: false}).pipe(s3(creds, {
delay: 1000,
headers: {
'x-amz-acl': 'public-read',
'Cache-Control': 'no-cache',
'Content-Type': 'text/html; charset=UTF-8',
// 'Cache-Control': 'no-transform,public,max-age=0,s-maxage=300', // NOTE: s-maxage is small for now, we could bump this up later once confident in cloudflare's cache purge workflow
},
makeUploadPath: makeUploadPathHtml,
}));
}

var tasks = process.argv.slice(2);
gulp.start.apply(gulp, tasks);
Expand Down
3 changes: 3 additions & 0 deletions package.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -26,16 +26,19 @@
"eslint-plugin-filenames": "^0.1.2",
"eslint-plugin-react": "^3.5.1",
"express": "^4.13.3",
"glob": "^7.1.2",
"gulp": "^3.9.1",
"gulp-gzip": "0.0.4",
"gulp-rename": "^1.0.0",
"gulp-s3": "git://github.com/mbjorkegren/gulp-s3.git#847e2b372204b4ceeccb0e9b567f28ff00e7fa1d",
"gulp-scp2 ": "^0.2.0",
"json-loader": "^0.5.1",
"react-transform-catch-errors": "^1.0.0",
"react-transform-hmr": "^1.0.1",
"redbox-react": "1.1.1",
"rimraf": "^2.4.3",
"run-sequence": "~0.3.6",
"scp2": "^0.5.0",
"webpack": "^1.12.2",
"webpack-dev-middleware": "^1.2.0",
"webpack-hot-middleware": "^2.4.1"
Expand Down
14 changes: 14 additions & 0 deletions polis.config.template.js
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,14 @@
module.exports = {

S3_BUCKET_PROD: 'pol.is',
S3_BUCKET_PREPROD: 'preprod.pol.is',

SCP_SUBDIR_PREPROD: 'preprod',
SCP_SUBDIR_PROD: 'prod',

//SERVICE_URL: 'http://localhost:5000',
SERVICE_URL: 'https://preprod.pol.is',

// UPLOADER: 'scp',
UPLOADER: 's3',
};

0 comments on commit 64056d0

Please sign in to comment.