The Orchestration namespace

Any arguments for this method will be in stages[stageIndex].instances.arg

Next stage is to get the data from the database. The count came from the previous stage, and GasThreader has calculated the 24 would be a good number of threads to run in parallel. This means that the count from the previous stage would have been used to distribute the work amongst each of the 24 threads. I have all my app specific code in the FusionToDb namespace so its not important to look at that. What is important is that it’s being told to retrieve a certain number of rows, starting at a particular position.

ns.getData = function (pack) {
    
    // get the reduced stage - all the data
    const stage = pack.stages[pack.stageIndex];
    const chunk = stage.chunks[pack.chunkIndex];
    
    // Important
    // if you need to modify arg - clone it first
    const arg = Utils.clone(stage.instances.arg);
    
    // in this case there's no data to get,
    // as the previous stage was just a count
    
    // arg contains the info needed to open the db
    // add the slice I'm doing
    arg.maxr = chunk.chunkSize;
    arg.start = chunk.start;
    
    // there's other chunking that happens in Fusion too
    // as there's a limit to how much can be read in one go
    // that's dealt with invisibly here
    const result = FusionToDB.getChunked (arg);

    // register it for future reference
    return Server.registerChunk (stage, chunk, result);
  
  };

{
          stage: "getData",
          stageTitle: "Getting data",
          skipReduce: true,

          // the maximum number of chunks to break the job into omit for the default
          maxThreads: 24,

          // the min size of a chunk to avoid splitting job into inefficiently small chunks
          // omit for default
          minChunkSize: 200,

          // the
          instances: {
            namespace: "Orchestration",
            method: "getData",
            arg: fusion
          }
        },

This stage is going to look at all the data to find out the maximum length of any data in them and deduce the data type to best use when creating the SQL table definition. This is fine to do many threads in parallel too, as I can combine the findings of each thread in a future merge stage.

 /**
  * we have to merge several definitions
  * @param {object} pack (stages, stageIndex, chunkIndex)
  * @return {object} updated chunk
  */
  ns.defineFields = function (pack) {
   
    // get the reduced stage - all the data
    const stage = pack.stages[pack.stageIndex];
    const chunk = stage.chunks[pack.chunkIndex];
    // we're going to enhance this arg, so take a copy
    const arg = Utils.clone(stage.instances.arg);
    
    // the results from the previous stage (or I could seeach for its name)
    const prev = pack.stages[pack.stageIndex -1];

    // normally prev.result would contain the reduced data from the previous stage 
    // but this takes care of the chunks being different sizes
    arg.data = Server.getSlice (prev ,chunk.start , chunk.chunkSize + chunk.start);
    
    // look at all the data and figure out the field types
    const result = FusionToDB.defineFields (arg);
    
    
    // register for this chunk
    return Server.registerChunk (stage, chunk, result, 1);
  
  };

Zip script files

Finally these scripts need to be combined into a single zip file, ready to be downloaded to their target server.

ns.makeSqlScripts = function (pack) {

    // get the reduced stage - all the data
    const stages = pack.stages;
    const stage = stages[pack.stageIndex];
    const chunk = stage.chunks[pack.chunkIndex];
    
    // we're going to enhance this arg, so take a copy
    const arg = Utils.clone(stage.instances.arg);
    
    // get the prev stage
    const prev = stages[stage.stageIndex -1];
    arg.sql =  Server.getSlice (prev ,chunk.start , chunk.chunkSize + chunk.start);
    
    // filename starts at chunk start + some offset if given
    arg.fileNameStart = (arg.fileNameStart || 0 ) + chunk.start ;
    arg.fileNameStem = (arg.fileNameStem || ("sqlins-" + arg.db + "-" + arg.table + "-") ) + zeroPad(chunk.chunkIndex,3) + "-";
    arg.fileExtension = arg.fileExtension || ".sql";

    arg.folderId = arg.folderId || DriveApp.getRootFolder().getId();
    var folder = DriveApp.getFolderById(arg.folderId);
    if (!folder) throw 'folder for id ' + arg.folderId + ' not found';

    const file = folder.createFile(folder.createFile(arg.fileNameStem+zeroPad(arg.fileNameStart,6)+arg.fileExtension, arg.sql.join("\n"), "text/plain"));


    // write that chunk
    return Server.registerChunk (stage, chunk, file.getId());
  };

This is the only tweaking you need to do to the Client side code (in red below), and I may improve this in future versions.

window.onload = function() {

  // set up client app structure
  App.initialize();
  Home.init();

  
  // get the ui going
  Client.init()
    .then (function (result) {
      // get the final result
      return Provoke.run("Server","getReduced",result.stage)
        .then (function (reduced) {
          Render.showResult (JSON.stringify(reduced.result));
          App.toast (result.work.title,"completed");
        })
    })
    ['catch'](function(err) {
      App.showNotification ("work failed",  err);
    });

};

Previous