Running things in parallel using HTML service was a brief intro on how to run a number of things at once, orchestrating executing using Google Apps Script HTML service. You may have already tried Parallel implementation and getting started or even Running things in parallel that need oAUTH2 – datastore. Now here’s something a little more complicated.
The process
Orchestration profile
Create the test data
I’m creating 4000 test items. 2 threads will do for this trivial process.
// first stage - create test data var scale = optScale || 2000, CHUNKS = 2; var profileTestData = []; for (var i =0; i <CHUNKS;i++ ) { profileTestData.push ({ name: 'testdata-'+i, functionName:'cryptoTestData', options:{ scale:scale } }); }
Reduce the results
// next reduce the test data to one var profileReduction = []; profileReduction.push({ name: 'reduction', functionName:'reduceTheResults', options:{} });
Do the encryption
// next,execute the cryptotests var CRYPTOTHREADS =2,CIPHERS=['tripledes','aes','rabbit','des']; var profileCrypto = []; for (var i = 0; i < CRYPTOTHREADS ; i++) { CIPHERS.forEach (function (d) { profileCrypto.push ({ name:'crypto-'+d+'-'+i, functionName:'cryptoEncrypt', options: { index: i, threads:CRYPTOTHREADS, cipher:d } }); }); }
Reduce the results
Check that it worked
// next we'll test the results by decrypting and check against original var DECRYPTOTHREADS = 6; var profileDecrypto = []; for (var i=0; i < DECRYPTOTHREADS ;i++ ) { profileDecrypto.push ({ name:'decrypto-'+i, functionName:'cryptoDecryptCheck', options: { stopOnFail:true, index:i, threads:DECRYPTOTHREADS } }); }
Reduce the results
Log the results
// finally log the results var profileLog = [{ "name": "LOG", "functionName": "cryptoLog", "skip": false, "options": { "driver": "cDriverSheet", "clear": true, "parameters": { "siloid": "logencrypt", "dbid": "1yTQFdN_O2nFb9obm7AHCTmPKpf5cwAd78uNQJiCcjPk", "peanut": "bruce" } } }];
Here’s what was logged
Put it all together
// put it all together profile.push ( profileTestData, profileReduction, profileCrypto, profileReduction, profileDecrypto, profileReduction ); if (cUseful.applyDefault(optLog, true) ) { profile.push(profileLog); }
The executor functions
- The options you set up in your profile
- The data from the previous stage (if there was any)
Creating the test data.
/** * create some test data for encryption tests * @param {object} options describes what to do * @param {object} reduceResults this would contain results from a previous stage if present * @return {object} test data to pass on to next stage */ function cryptoTestData (options, reduceResults) { var testData = []; //generate some test data for (var i=0; i < options.scale ; i++ ) { testData.push ( { message: cUseful.generateUniqueString(Math.round(Math.random()*100)) , password: cUseful.generateUniqueString(Math.round(Math.random()*10)) }); } return testData; }
Reduce the data
/** * reduce the results from a previous mapping excercise * @param {object} options describes what to do * @param {object} mapResults this would contain results from a previous stage if present * @return {array.*} test data to pass on to next stage */ function reduceTheResults(options, mapResults) { // we'll have all the results here so consolidate var results = mapResults.reduce ( function (p,c) { (Array.isArray (c.results) ? c.results : [c.results]).forEach (function(d) { p.push (d); }) return p; },[]); return results; }
Do some encryptions
/** * do a chunk of encryption testing * @param {object} options describes what to do * @param {object} reduceResults this would contain results from a previous stage if present * @return {object} test data to pass on to next stage */ function cryptoEncrypt (options,reduceResults) { // get testdata and encrypt // we'll only do a section of data in this thread var data = reduceResults[0].results; var start = options.index/options.threads * data.length; var finish = (options.index+1)/options.threads * data.length ; // encrypt that slice return data.slice (start, finish).map ( function (d) { // for later decrypt testing, we'll include everything return { encrypted:new cCryptoGS.Cipher(d.password,options.cipher).encrypt(d.message), password:d.password, message:d.message, cipher:options.cipher } }); }
Decrypt and check
/** * do a chunk of decryption testing * @param {object} options describes what to do * @param {object} reduceResults this would contain results from a previous stage if present * @return {object} test data to pass on to next stage */ function cryptoDecryptCheck (options,reduceResults) { // check that it all decrypts properly // we'll only do a section of data in this thread var data = reduceResults[0].results; var start = options.index/options.threads * data.length; var finish = options.threads === options.index + 1 ? data.length : (options.index+1)/options.threads * data.length ; return data.slice(start,finish).map ( function (d) { var decrypt = new cCryptoGS.Cipher(d.password,d.cipher).decrypt(d.encrypted); if (options.stopOnFail) { if(decrypt !== d.message) throw 'decrytped does not match encrypted' + d.cipher; } return {cipher:d.cipher, error: decrypt !== d.message}; }); }
Logging the results
/** * summarize the results of the orchestration * @param {object} options describes what to do * @param {object} reduceResults this would contain results from a previous stage if present * @return {object} test data to pass on to next stage */ function cryptoLog (options,reduceResults) { // load it to crossFilter var cf = cCrossFilter.crossfilter(reduceResults[0].results); // create a cipher dimension var ciphers = cf.dimension ( function (d) { return d.cipher }); // write a summary var cipherSummary = ciphers.filter(null).group().all(); var handler = new cDbAbstraction.DbAbstraction ( eval(options.driver), options.parameters ); assert(handler.isHappy(), 'unable to get handler',options.driver); if (options.clear) { var result = handler.remove(); if (result.handleCode < 0) { throw result.handleError; } } var result = handler.save(cipherSummary); if (result.handleCode < 0) { throw result.handleError; } return reduceResults.results; }
The whole profile
/** * Shows a custom HTML user interface in a sidebar */ function showSidebar() { // kicking off the sidebar executes the orchestration libSidebar('asyncService',ADDONNAME, cryptoProfile () ); }
Here’s the code for creating the profile for this all together.
function cryptoProfile() { // profile for testing crypto functions var profile = []; // first stage - create test data var SCALE = 2000, CHUNKS = 2; var profileTestData = []; for (var i =0; i <CHUNKS;i++ ) { profileTestData.push ({ name: 'testdata-'+i, functionName:'cryptoTestData', options:{ scale:SCALE } }); } // next reduce the test data to one var profileReduction = []; profileReduction.push({ name: 'reduction', functionName:'reduceTheResults', options:{} }); // next,execute the cryptotests var CRYPTOTHREADS =2,CIPHERS=['tripledes','aes','rabbit','des']; var profileCrypto = []; for (var i = 0; i < CRYPTOTHREADS ; i++) { CIPHERS.forEach (function (d) { profileCrypto.push ({ name:'crypto-'+d+'-'+i, functionName:'cryptoEncrypt', options: { index: i, threads:CRYPTOTHREADS, cipher:d } }); }); } // do another reduction - we can use the same reduction profile, so nothing to do here // next we'll test the results by decrypting and check against original var DECRYPTOTHREADS = 6; var profileDecrypto = []; for (var i=0; i < DECRYPTOTHREADS ;i++ ) { profileDecrypto.push ({ name:'decrypto-'+i, functionName:'cryptoDecryptCheck', options: { stopOnFail:true, index:i, threads:DECRYPTOTHREADS } }); } // do another reduction - we can use the same reduction profile, so nothing to do here // finally log the results var profileLog = [{ "name": "LOG", "functionName": "cryptoLog", "skip": false, "options": { "driver": "cDriverSheet", "clear": true, "parameters": { "siloid": "logencrypt", "dbid": "1yTQFdN_O2nFb9obm7AHCTmPKpf5cwAd78uNQJiCcjPk", "peanut": "bruce" } } }]; // put it all together profile.push ( profileTestData, profileReduction, profileCrypto, profileReduction, profileDecrypto, profileReduction, profileLog ); return profile; }
and here is the profile it produces
[ [ { "name": "testdata-0", "functionName": "cryptoTestData", "options": { "scale": 2000 } }, { "name": "testdata-1", "functionName": "cryptoTestData", "options": { "scale": 2000 } } ], [ { "name": "reduction", "functionName": "reduceTheResults", "options": {} } ], [ { "name": "crypto-tripledes-0", "functionName": "cryptoEncrypt", "options": { "index": 0, "threads": 2, "cipher": "tripledes" } }, { "name": "crypto-aes-0", "functionName": "cryptoEncrypt", "options": { "index": 0, "threads": 2, "cipher": "aes" } }, { "name": "crypto-rabbit-0", "functionName": "cryptoEncrypt", "options": { "index": 0, "threads": 2, "cipher": "rabbit" } }, { "name": "crypto-des-0", "functionName": "cryptoEncrypt", "options": { "index": 0, "threads": 2, "cipher": "des" } }, { "name": "crypto-tripledes-1", "functionName": "cryptoEncrypt", "options": { "index": 1, "threads": 2, "cipher": "tripledes" } }, { "name": "crypto-aes-1", "functionName": "cryptoEncrypt", "options": { "index": 1, "threads": 2, "cipher": "aes" } }, { "name": "crypto-rabbit-1", "functionName": "cryptoEncrypt", "options": { "index": 1, "threads": 2, "cipher": "rabbit" } }, { "name": "crypto-des-1", "functionName": "cryptoEncrypt", "options": { "index": 1, "threads": 2, "cipher": "des" } } ], [ { "name": "reduction", "functionName": "reduceTheResults", "options": {} } ], [ { "name": "decrypto-0", "functionName": "cryptoDecryptCheck", "options": { "stopOnFail": true, "index": 0, "threads": 6 } }, { "name": "decrypto-1", "functionName": "cryptoDecryptCheck", "options": { "stopOnFail": true, "index": 1, "threads": 6 } }, { "name": "decrypto-2", "functionName": "cryptoDecryptCheck", "options": { "stopOnFail": true, "index": 2, "threads": 6 } }, { "name": "decrypto-3", "functionName": "cryptoDecryptCheck", "options": { "stopOnFail": true, "index": 3, "threads": 6 } }, { "name": "decrypto-4", "functionName": "cryptoDecryptCheck", "options": { "stopOnFail": true, "index": 4, "threads": 6 } }, { "name": "decrypto-5", "functionName": "cryptoDecryptCheck", "options": { "stopOnFail": true, "index": 5, "threads": 6 } } ], [ { "name": "reduction", "functionName": "reduceTheResults", "options": {} } ], [ { "name": "LOG", "functionName": "cryptoLog", "skip": false, "options": { "driver": "cDriverSheet", "clear": true, "parameters": { "siloid": "logencrypt", "dbid": "1yTQFdN_O2nFb9obm7AHCTmPKpf5cwAd78uNQJiCcjPk", "peanut": "bruce" } } } ] ]
or more snippets like this see Google Apps Scripts snippets