Implementing a client side progress bar reporting on server side work for htmlService


It's always a compromise to use a progress bar, as it's usually not known how long something will take, and therefore how to plot progress. This is especially true in Apps Script when the activity is happening on the server, disconnected from the client side that is trying to report on its progress. In addition, server side activities don't tend to have many break points, since they are synchronous.  

 In today's post, I'll show you how to create a fairly smooth material design type progress bar that runs client side and reports on things that are happening client side. For the example, I'll use an updated version of the code described in A webapp to share copies of the contents of folders. Although there's a fair bit of code here, and some of the problems around asynchronicity and polling are quite challenging,  most of it is boiler plate.

It looks like this 


With a discrete progress bar at top of screen, a spinner, and also the capability to return arbitrary objects which can be used to report on the status of things. The example here is able to update the table with the status of copying files at the same time as drawing a progress bar.
You can get it to copy itself to your drive using http://goo.gl/p3Ixbl

How it works

 I'm using this very simple nice bar renderer on the client side - http://ricostacruz.com/nprogress/ 

On the server side, cache is used to store progress as the activity is looped through. The normal server side loop activity contains some code to write some progress info at every opportunity, and is shown in red. This is interrogated from the client side, which passes  (the item parameter), the latest status update it saw. Like this, data can be persisted across asynchronous client/server calls. Note that the server side doesn't return the status update (it would be too late as it would be finished by then). We'll look at how the client side gets that later.
ns.copyFiles = function (data,item) {
    
    // get the source and target folders
    var sourceFolder = DriveApp.getFolderById(data.source.id);
    var targetFolder = DriveApp.getFolderById(data.target.id);
    
    // we can use this to send back what we're actually working on
    item.activeFiles = [];
    
    // for each of the data files make a copy to the target folder
    data.files.forEach(function (d,i,a) {
      var file = DriveApp.getFileById(d.id);
      if (item) {
        // set max an min progress for this pass
        item.activeFiles.push(d.id);
        item.max = (i+1)/a.length;
        item.min = i/a.length;
        ns.setProgress(item.min, item);
      }
      // copy the file
      file.makeCopy(file.getName(), targetFolder);

    });
    // done
    if (item) {
      ns.setProgress(1, item);
    }    
    // return the data that was sent for processing
    return data;
  };
Note that progress is a number between 0-1 that shows how far through the task the server has got. But if there is only one or two files to copy, it wouldn't be possible to show a smooth progress on the client side from only a couple of data points. That's why the min and max values are also given, and they tell the client that this report lies somewhere between min and max. When the client retrieves this status report it knows the range it should be in by using min and max, and adjusts it's plot rate according to how long it's taken to get this far versus how long we still have to go, which it can deduce from the min and max.

There are a couple of other things needed on in the Server side to manage creating the status reports. None of this needs modification unless you want to add extra fields to communicate.
var getCache_ = function () {
    return CacheService.getUserCache();
  };
  /**
   * initialize a server progress item
   * @param {object} [item] an initial value for the progress item if required
   * @return {object} a status object
   */
  ns.createProgress = function (item) {
    return ServerProgress.newItem(getCache_(),item).getStatus();
  };
  
    /**
   * initialize a server progress item
   * @param {object} [item] an initial value for the progress item if required
   * @return {object} a status object
   */
  ns.getProgress = function (item) {
    return ServerProgress.getById (getCache_(),item.id);
  };
  
  /**
   * set progress
   * @param {number} progress progress 0-1
   * @param {object} item the current item
   * @return {object} the updated item
   */
  ns.setProgress = function (progress,item) {
    return ServerProgress.setItem (getCache_(), item, progress);
  };

as well as this namespace, which manages cache.
/**
*@namespace ServerProgress
* this is a progress tracker
* that runs server side
* and should be called from client side 
* periodically
*/
var ServerProgress = (function(ns) {
  
  /**
  * @constructor ProgressItem
  */
  var ProgressItem = function (cache,initialItem, id,  progress) {
    
    var self = this, cache_=cache, id_= id || Utilities.getUuid(), initial_ = initialItem || null;
    
    /**
    * get the id of this progress item
    * @return {string} the id
    */
    self.getId = function () {
      return id_;
    };
    
    /**
    * update progress
    * @param {number} progress updated progress 0-1.
    * @param {object} [statusObject] previous status
    * @return {object} updated status object 
    */
    self.setStatus = function ( progress, statusObject ) {
      
      statusObject = statusObject || {};
      statusObject.updatedAt = new Date().getTime();
      statusObject.startTime = statusObject.startTime || statusObject.updatedAt;
      statusObject.count = (statusObject.count || 0) + 1;
      statusObject.id = statusObject.id || id_;
      statusObject.progress = progress;
      
      cache_.put (id_ , JSON.stringify(statusObject) ,60*60);
      return statusObject;  
    };
    
    /**
    * get status
    * @return {object} status 
    */
    self.getStatus = function () {
      return ns.getById(cache_, id_);
    };
    
    // initialize 
    self.setStatus (progress || 0,initial_);
    
  };
  
  
  /**
  * generate a new progress Item
  * @param {cache} cache the cache to use
  * @param {object} [initialItem] and intial item to carry around if required
  * @return {ProgressItem} a progress item
  */
  ns.newItem = function (cache,initialItem) {
    return new ProgressItem(cache,initialItem);
  };
  
  /**
  * get a status given a progress item id
  * @param {cache} cache the cache to find it in
  * @param {string} id the id of the progres item
  * @return {ProgressItem} the progress item
  */
  ns.getById = function (cache, id) {
    var status = cache.get (id);
    return status ? JSON.parse(status) : null;
  };
  
  /**
  * set a status given a progress item id
  * @param {cache} cache the cache to find it in
  * @param {object} status what to set it to
  * @param {number} progress the progress to set it to
  * @return {ProgressItem} the progress item
  */
  ns.setItem = function (cache, status, progress) {
    return new ProgressItem(cache, status , status.id, progress).getStatus();
  };
  
  return ns;
})(ServerProgress || {});

Client Side

The client side polls the Server to ask for status updates according to its own schedule, and plots the progress bar, also to its own schedule, until it detects a change from the server side from which it can make a better estimate of the finish time. It uses ifvisible, to ensure that its only bothering to poll if anybody is actually looking at it.  None of this needs changed either, unless you want to change the polling frequency, which is controlled here
  ns.control = {
    progress:{
      status:null,
      interval:750
    },

Client side polling  

/**
  * poll for progress update on the server
  */
  ns.progressPoll = function (immediate, close) {
    
    if (ClientProgress.control.running && ifvisible.now()) {
      poller_(immediate ? 10 : ns.control.progress.interval)
      
      .then (function () {
        return Provoke.run('Server','getProgress', ns.control.progress.status);
      })
      
      .then (function (status) {
        
        // if its still running after all that
        if (ClientProgress.control.running) {
          // store the status
          ns.control.progress.status = status;
        
          // update the status bar
          ClientProgress.smoothProgress (ns.control.progress.status);

          // update any files that are finished
          ns.updateFileProgress(close);
          
          // the last one finished with a close down
          if (close) {
            ClientProgress.done();
          }
        }
        
        // recurse - if closed it wont run.
        ns.progressPoll();

      })
      
      ['catch'](function (err) {
        App.showNotification(err);
      });
    }
    // wait a bit longer and try again
    else {
      poller_(ns.control.progress.interval*(1+Math.random()))
      .then (function() {
        ns.progressPoll();
      })
    }

  };
  
  // polling for progress update
  function poller_ (waitFor) {
    
    return new Promise (function (resolve, reject) {
      setTimeout(function () {
        resolve();
      },waitFor);
    });
    
  }

Controlling the progress bar

This namespace  dynamically smooths the progress bar- given that the server updates are not going to be very regular, and shouldn't need any tweaking. You can see here how the item.max and item.min are used to make sure that the progress bar stays in reasonable bounds, and how the estimated overall duration is adjusted by how long things have taken so far.
/**
* runs client side and
* coperates with ServerProgress
* to report on progress being made server side
* @namespace ClientProgress
*/
var ClientProgress = (function (ns) {
  
  ns.control = {
    running:false,
    stuck:{
      count:0
    },
    trickle: {
      duration:0,
      speed:250,
      defaultDuration:200000
    }
  };
  
  // check if the bar is running
  ns.isRunning = function () {
    return NProgress.isStarted()
  };
  
  /** 
   * set the trickle speed
   * every time we have a new 
   * estimate for how long this will take
   * @param {number} duration new estimated duration in ms
   * @return {ClientProgress} self
   */
  ns.setTrickle = function (duration) {
  
    // this is the expected overall duration 
    // and affects the trickle speed and rate.
    var nt = ns.control.trickle;
    
    // the rate at which it needs to go it whats left - whats been reported
    var rate = nt.speed/(ns.isRunning() ? (1-ns.getBarProgress())*duration : duration) ;
    nt.duration = duration;
    
    // adjust the trickle rate to accommodate whats left
    NProgress.configure({ 
      trickleRate: rate , 
      trickleSpeed: nt.speed 
    });
    return ns;
    
  };
  
  // return current status bar position
  ns.getBarProgress = function () {
    return NProgress.status || 0;
  };
  
  // start the bar
  ns.start = function () {
    ns.control.running = true;
    ns.control.stuck.count = 0;
    ns.setTrickle (ns.control.trickle.defaultDuration);
    return NProgress.start();
  };
  
  // set to a particular value
  ns.set = function (p) {
    return NProgress.set(p);
  };
  
  // its all over
  ns.done = function () {
    ns.control.running = false;
    return NProgress.done();
  };
  
  // increment a little bit
  ns.inc = function () {
    return NProgress.inc();
  };
  
  // any initialization
  ns.initialize = function () {
    NProgress.configure({ 
      easing: 'linear',
      trickleRate:.01,
      trickleSpeed:ns.control.trickle.speed
    });
    return ns;
  };
  
  /**
   * smooths out progress according to data in item
   * @param {object} item the item generated server side
   * @return {ClientProgress} self
   */
  ns.smoothProgress = function (item) {
    
    var nt = ns.control.trickle;
    
    // we have the starttime, and the update time. so that's the duration to get this far
    var duration = item.updatedAt - item.startTime;
    
    // how long it'll take at this rate
    var estDuration = item.progress ? duration /item.progress : nt.duration;

    // so the expected duration must be between 
    var soFar = nt.duration * ns.getBarProgress();
    var maxDuration = duration / Math.min(item.max,1);
    var minDuration = duration / Math.max(item.min,0.01);

    
    // we can set the new estimate duration if needed
    if (soFar > maxDuration && item.max) {
       ns.setTrickle (maxDuration);
    }
    else if (soFar < minDuration && item.min) {
       ns.setTrickle (minDuration);
    }
    else if (item.progress) {
       ns.setTrickle (estDuration);
    }
    
    return ns;
    
  };
  
  
  
  return ns;
})(ClientProgress || {});


Initializing the server side progress

This creates a progress recording environment on the server side. It of course happens asynchonously. I use my own Provoke.run to do this, but you can of course use google.script.run as well.
    // kick off getting the files and creating a server progress session
    var pProg = Provoke.run("Server","createProgress");
    
    // when createprogress is done then store the item
    pProg.then (function(status) {
      ns.control.progress.status = status;
    })
    ['catch'](function(err) {
      App.showNotification ("Progress bar problem", err);
    });


Starting and stopping the progress bar


In my App, the progress bar kicks off whenever the user hits 'copy', and the progress related code is shown in red
// get set up listeners and get started
  ns.initialize = function () {
    
    // add event listeners
    DomUtils.elem("copy").addEventListener("click", function() {
      // starting the copy
      DomUtils.elem("copy").disabled = true;
      
      // start the progress
      if (ns.control.progress.status) {
        
        // initialize progress options and go
        ClientProgress
        .initialize()
        .start();
        
        // recurse immediately
        ns.progressPoll(true);
      }
      
      // get the server going, copying files.
      Provoke.run("Server", "copyFiles", result,ns.control.progress.status)
      .then(function(data) {
        
        // recurse immediately, and then close
        ns.progressPoll(true, true);
        
        // all done
        App.toast('Copy completed', data.files.length +
                  ' files copied to folder ' + data.target.name);
        
        
      })['catch'](function(err) {
        DomUtils.elem("copy").disabled = false;
        App.showNotification("Copy failure", err);
        
        // close down the progress bar
        ClientProgress.done();
      });
      
    }, false);
    return refreshData();
  };be p



You want to learn Google Apps Script?

Learning Apps Script, (and transitioning from VBA) are covered comprehensively in my my book, Going Gas - from VBA to Apps script, available All formats are available now from O'Reilly,Amazon and all good bookshops. You can also read a preview on O'Reilly

If you prefer Video style learning I also have two courses available. also published by O'Reilly.
Google Apps Script for Developers and Google Apps Script for Beginners.





Extra parameters

You'll notice that I'm dynamically updating the table with which file is currently being copied.


That's because the progress process can also be used to persist other data in addition to progress data. Back in the server side function that is being measured, I'm maintaining an array - activeFiles. This is being used to communicate all the files as they get done.

ns.copyFiles = function (data,item) {
    
    // get the source and target folders
    var sourceFolder = DriveApp.getFolderById(data.source.id);
    var targetFolder = DriveApp.getFolderById(data.target.id);
    
    // we can use this to send back what we're actually working on
    item.activeFiles = [];
    
    // for each of the data files make a copy to the target folder
    data.files.forEach(function (d,i,a) {
      var file = DriveApp.getFileById(d.id);
      if (item) {
        // set max an min progress for this pass
        item.activeFiles.push(d.id);
        item.max = (i+1)/a.length;
        item.min = i/a.length;
        ns.setProgress(item.min, item);
      }
      // copy the file
      file.makeCopy(file.getName(), targetFolder);

    });
    // done
    if (item) {
      ns.setProgress(1, item);
    }    
    // return the data that was sent for processing
    return data;
  };

And back on the client side,at each polling event,  I just do this to report on the file status in my  table
  /**
   * there's a package that comes with the status
   * showing which files have actually been completed
   * @param {boolean} done its all done
   * @return {Client} self
   */
  ns.updateFileProgress = function (done) {
  
    var nc = ns.control;
    var status = nc.progress.status;
    
    var index = VizTable.getColumnIndex (nc.table.dataTable, "copy status");
    
    if (index < 0) {
      App.showNotification ("Table error", "couldnt retrieve statux index");
      ClientProgress.done();
    }
    else {
      //do something
      (status.activeFiles || []).forEach(function (d,i,a) {
        nc.table.dataTable.setCell (i , index , done || i < a.length-1 ? "done" : "copying");
        VizTable.draw  (nc.table.dataTable, nc.table.vizTable);
      });
    }
   
  };


The code


You can get your copy delivered to your Drive with the webApp using  http://goo.gl/p3Ixbl

This blogpost gives some more info on setting up for your own folders.

For more like this, see Google Apps Scripts snippets. Why not join our forum, follow the blog or follow me on twitter to ensure you get updates when they are available.





Comments