This is the the driver for DB.DATASTORE described in Database abstraction with google apps script
The library reference is
MPZF_EC6nOZFAjMRqCxEaUyz3TLx7pV4j
This is a little more complicated that the other drivers, and in fact is made up of 3 different classes
Here is the code consisting of 3 modules
/**TODO
* immediate queries after write are too quick .. update not yet done - maybe transactional is needed
* indexes required for filter/sort combination
*/
/** wrapper
*/
function createDriver (handler,siloId,driverSpecific,driverOb, accessToken) {
return new DriverDataStore(handler,siloId,driverSpecific,driverOb, accessToken);
}
/** TODO
* having problems with 412 and predefined indexes
*/
function getLibraryInfo () {
return {
info: {
name:'cDriverDataStore',
version:'2.0.1',
key:'MPZF_EC6nOZFAjMRqCxEaUyz3TLx7pV4j',
share:'https://script.google.com/d/1gKZkk4zuouPmIf1JYAFGTfCW0AmMtbL5eTohuLmcOE2WqIDxLudAMrxB/edit?usp=sharing',
description:'cloud datastore driver for dbabstraction'
},
dependencies:[
cFlatten.getLibraryInfo()
]
};
}
/**
* DriverDataStore
* @param {cDataHandler} handler the datahandler thats calling me
* @param {string} tableName this is the DataStore collection
* @param {string} id some id you create for identifying this collection
* @param {object} dataStoreOb a DataStoreOb ob if required ( { restAPIKey:"your DataStore developer key"} )
* @param {string} [accessToken] an oauth2 access token
* @return {DriverDataStore} self
*/
var DriverDataStore = function (handler,tableName,id,dataStoreOb,optAccessToken) {
var siloId = tableName; // the kind
var dbId = id; // cloud datastore project id
var self = this;
var parentHandler = handler;
var enums = parentHandler.getEnums();
var keyOb = dataStoreOb;
var handleError, handleCode;
var accessToken = optAccessToken || null;
var handle = new DatastoreWorker(parentHandler, accessToken, dbId,siloId);
var complex_ = false;
// im not able to do transactions
self.transactionCapable = false;
// i need transaction locking
self.lockingBypass = false;
// i am aware of transactions and know about the locking i should do
self.transactionAware = true;
self.getType = function () {
return enums.DB.DATASTORE;
};
self.getDbId = function () {
return dbId;
};
/** no persistent handle for this rest query - just return self
* @return {DriverDataStore} self
*/
self.getDriveHandle = function () {
return handle;
};
/**
* DriverDataStore.getTableName()
* @return {string} table name or silo
*/
self.getTableName = function () {
return siloId;
};
/** create the driver version
* @return {string} the driver version
*/
self.getVersion = function () {
var v = getLibraryInfo().info;
return v.name + ':' + v.version;
};
/**
* DriverDataStore.query()
* @param {object} queryOb some query object
* @param {object} queryParams additional query parameters (if available)
* @param {boolean} keepIds whether or not to keep driver specifc ids in the results
* @return {object} results from selected handler
*/
self.query = function (queryOb,queryParams,keepIds) {
return parentHandler.readGuts ( 'query' , function() {
return queryGuts_(queryOb,queryParams,keepIds);
});
};
function queryGuts_(queryOb,queryParams,keepIds) {
var result,driverIds, handleKeys;
handleCode = enums.CODE.OK, handleError='',qp=null;
complex_ = false;
if (queryParams) {
result= self.sortOutParams(queryOb,queryParams);
if (result.handleCode !== enums.CODE.OK) return result;
qp = result.data;
}
result = handle.runQuery(queryOb,qp);
if (result.handleCode !== enums.CODE.OK) {
return parentHandler.makeResults (result.handleCode,result.handleError);
}
// apply anyfilters
var pr = parentHandler.processFilters (queryOb, result.data);
handleCode =pr.handleCode;
handleError = pr.handleError;
if (handleCode === enums.CODE.OK) {
result.data = pr.data;
// fix up the result
if (keepIds) {
var d = handle.getIds(result);
driverIds = pr.handleKeys.map(function(k) {
return d[k];
});
handleKeys= driverIds;
}
}
// next fix up parameters datastore couldnt handle
if (complex_ ) {
var npr = parentHandler.processParams( queryParams,result.data);
if (handleCode === enums.CODE.OK) {
result.data = npr.data;
// fix up the result
if (keepIds) {
driverIds = npr.handleKeys.map(function(k) {
return driverIds[k];
});
handleKeys= driverIds;
}
}
}
return parentHandler.makeResults (handleCode,handleError,result.data,keepIds ? driverIds :null,keepIds ? handleKeys:null);
}
/**
* DriverDataStore.removeByIds()
* @memberof DriverDataStore
* @param {Array.string} ids list of handleKey ids to remove
* @return {object} results from selected handler
*/
self.removeByIds = function (ids) {
var result = {};
try {
Logger.log('removing by ids');
Logger.log(ids);
result = handle.remove (ids);
Logger.log(result);
}
catch(err) {
result.handleError = err;
result.handleCode = enums.CODE.DRIVER;
}
return parentHandler.makeResults (result.handleCode,result.handleError,result);
};
/**
* DriverDataStore.remove()
* @param {object} queryOb some query object
* @param {object} queryParams additional query parameters (if available)
* @return {object} results from selected handler
*/
self.remove = function (queryOb,queryParams) {
return parentHandler.writeGuts ( 'remove' , function() {
try {
var result;
try {
// start with a query
result = queryGuts_ (queryOb, queryParams , true);
if (result.handleCode === enums.CODE.OK) {
result = self.removeByIds (result.handleKeys);
}
}
catch(err) {
result = {
handleError: err,
handleCode: enums.CODE.DRIVER
};
}
return parentHandler.makeResults (result.handleCode,result.handleError,result);
}
catch (err) {
return parentHandler.makeResults(enums.CODE.LOCK,err);
}
});
};
/**
* DriverDataStore.save()
* @param {Array.object} obs array of objects to write
* @return {object} results from selected handler
*/
self.save = function (obs) {
return parentHandler.writeGuts ( 'save' , function() {
var result = null;
handleError='', handleCode=enums.CODE.OK;
if(handleCode === enums.CODE.OK) {
try {
result = handle.insert(obs);
if (result.handleCode !== enums.CODE.OK) {
handleError = result.handleError;
handleCode = result.handleCode;
}
}
catch(err) {
handleError = err ;
handleCode = enums.CODE.DRIVER;
}
}
return parentHandler.makeResults (handleCode,handleError,result);
});
};
/**
* DriverDataStore.count()
* @param {object} queryOb some query object
* @param {object} queryParams additional query parameters (if available)
* @return {object} results from selected handler
*/
self.count = function (queryOb,queryParams) {
return parentHandler.readGuts ( 'count' , function() {
var c=0, result= self.query(queryOb,queryParams);
if(result.handleCode >= 0) {
c = result.data.length;
}
return parentHandler.makeResults (handleCode,handleError,[{count:c}]);
});
};
/**
* Driver.get()
* @param {string} key the unique return in handleKeys for this object
* @param {boolean} keepIds whether or not to keep driver specifc ids in the results
* @return {object} results from selected handler
*/
self.get = function (key,keepIds) {
return parentHandler.readGuts ( 'get' , function() {
var result =null;
handleError='', handleCode=enums.CODE.OK;
var driverIds, handleKeys;
try {
result = handle.lookup(key);
if (result.handleCode !== enums.CODE.OK) {
handleError = result.handleError;
handleCode = result.handleCode;
}
}
catch(err) {
handleError = err ;
handleCode = enums.CODE.DRIVER;
}
// fix up the result
if (keepIds) {
driverIds= handle.getIds(result);
handleKeys= driverIds;
}
return parentHandler.makeResults (handleCode,handleError,result.data,keepIds ? driverIds :null,keepIds ? handleKeys:null);
});
};
/**
* Driver.update()
* @param {string} key the unique return in handleKeys for this object
* @param {object} ob what to update it to
* @return {object} results from selected handler
*/
self.update = function (key,ob) {
return parentHandler.writeGuts ( 'update' , function() {
try {
var result;
try {
// do the update
result = handle.update(key,ob);
}
catch(err) {
result = {
handleError: err,
handleCode: enums.CODE.DRIVER
};
}
return parentHandler.makeResults (result.handleCode,result.handleError,result);
}
catch (err) {
return parentHandler.makeResults(enums.CODE.LOCK,err);
}
});
};
/**
* sort out query params
* @param {object} queryParams parameters
* @return
*/
self.sortOutParams = function (queryOb,queryParams) {
var pOb = null;
if (queryParams) {
// we cant rely on datastore to do limits and skips if there are complex queries with no index
complex_ = queryOb && Object.keys(queryOb).some (function(k) {
return (parentHandler.isObject (queryOb[k]) );
});
var result = parentHandler.getQueryParams(handle.flatten(queryParams));
if(result.handleCode === enums.CODE.OK) {
pOb = result.data.reduce (function (p,c) {
if (c.param === 'sort') {
p.order = [{property:{name:c.sortKey},direction: c.sortDescending ? 'DESCENDING' : 'ASCENDING' }];
}
else if (!complex_ && c.param === 'limit' ) {
p.limit = c.limit;
}
else if (!complex_ && c.param === 'skip') {
p.offset = c.skip;
}
return p; }, {});
return parentHandler.makeResults (enums.CODE.OK,'',pOb);
}
else {
return result;
}
}
return (enums.CODE.OK);
}
return self;
}
/**
* interface for data store - takes care of accessing datastore
* @param {DataHandler} parentHandler the abstract handker
* @param {string} accessToken - the oauth2 access token
* @param {string} dataStoreName - the name of this data store (project id incloud console)
* @param {string} kind - somewhat like the table name
* @return {DatastoreWorker} self
* @class DatastoreWorker
*/
function DatastoreWorker(parentHandler, accessToken, dataStoreName, kind) {
var name_ = dataStoreName;
var self_ = this;
var kind_ = kind;
var accessToken_ = accessToken;
var parentHandler_ = parentHandler;
var enums = parentHandler.getEnums();
var SAVECHUNK = 500;
self_.getParentHandler = function () {
return parentHandler_;
}
/**
* set a new access token
* @param {string} accessToken - the oauth2 access token
* @return {DatastoreWorker} self
*/
self_.setAccessToken = function (token) {
accessToken_ = token;
return self_;
};
/**
* flatten an array of objects/a single object
* @param {Array.Object|Object} obs an array of/single unflattened objects
* @param {boolean} optConstraints whether there might be constraints to preserve
* @return {Array.Object|Object} an array of/single flattened objects
*/
self_.flatten = function (obs,optConstraints) {
return self_.getParentHandler().flatten(obs,optConstraints);
};
/**
* unflatten an array of objects
* @param {Array.Object} obs an array of flattened objects
* @return {Array.Object} and array of unflattened objects
*/
self_.unflatten = function (obs) {
// unflatten the query
if (!obs) return null;
return obs.map(function(d) {
return new cFlatten.Flattener().unFlatten(d);
});
};
/**
* generate a proper object from a datastore entity query response
* @param {Array.object} entities - the datastore query response
* @return {Array.object} an array of proper objects
*/
self_.reconstructProperties = function (entities) {
var root;
if(entities.batch) {
root = entities.batch.entityResults;
}
else if (entities.found) {
root = entities.found;
}
else if (entities.entityResults) {
root = entities.entityResults;
}
else {
throw 'cant make anything of ' + JSON.stringify(entities);
}
return root.map( function(d) {
return new DataStoreEntity (d.entity.key.path[0].kind).injectProperties(d.entity.properties).reconstructProperties();
});
};
/**
* insert an array of objects
* @param {Array.object} obs - an array of objects to insert
* @return {object} the datastore result object and success codes
*/
self_.insert = function (obs) {
var result = {handleCode:enums.CODE.OK,handleError:''};
if (!Array.isArray (obs)) obs = [obs];
var entities = obs.map(function(d){
return new DataStoreEntity (kind_).setProperties (self_.flatten(d));
});
var options = getOptions_(), done =0;
while (done < entities.length && result.handleCode === enums.CODE.OK) {
var chunk = entities.slice(done, Math.min(entities.length,done+SAVECHUNK));
options.payload= JSON.stringify({
mutation: {insertAutoId: chunk.map ( function (d) { return d.objectify()})},
mode: "NON_TRANSACTIONAL"
});
result = execute_ (self_.getEndpoint("commit"), options);
done += chunk.length;
}
return result;
};
/**
* delete an array of objects
* @param {Array.string} obs - an array of ids to delete
* @return {object} the datastore result object and success codes
*/
self_.remove = function (ids) {
var result = {handleCode:enums.CODE.OK,handleError:''};
var options = getOptions_(),done =0;
while (done < ids.length && result.handleCode === enums.CODE.OK) {
var t = new DataStoreEntity (kind_);
var chunk = ids.slice(done, Math.min(ids.length,done+SAVECHUNK));
options.payload = JSON.stringify({
mutation: {"delete": t.removeify(chunk)},
mode: "NON_TRANSACTIONAL"
});
result= execute_ (self_.getEndpoint("commit"), options);
done += chunk.length;
}
return result;
};
/**
* update an array of objects
* @param {Array.string} obs - an array of ids to update
* @param {Array.object} obs - an array of objects to update it to
* @return {object} the datastore result object and success codes
*/
self_.update = function (ids,obs) {
if (!Array.isArray (obs)) obs = [obs];
if (!Array.isArray (ids)) ids = [ids];
if(ids.length !== obs.length) {
return {
handleCode: enums.CODE.KEYS_AND_OBJECTS,
handleError: 'objects- ' + obs.length + ' keys- ' + ids.length,
result:null };
}
else {
var entities = obs.map(function(d){
return new DataStoreEntity (kind_).setProperties (self_.flatten(d));
});
var options = getOptions_();
options.payload= JSON.stringify({
mutation: {update: entities.map ( function (d,i) { return d.updateify(ids[i])})},
mode: "NON_TRANSACTIONAL"
});
var result= execute_ (self_.getEndpoint("commit"), options);
if (result.handleCode >=0) {
result.data = self_.unflatten (result.mutationResult);
}
return result;
}
};
self_.getConstraintName = function (constraint) {
return constraint ?
Object.keys(enums.CONSTRAINTS).reduce (function(p,c) {
return enums.CONSTRAINTS === constraint ? enums.DATASTORE_CONSTRAINTS : p;
},'')
: null;
}
/**
* run a query
* @param {object} optOb - a nosql query
* @param {optParams} optParams - sorted out parameters
* @return {object} the datastore result object and success codes
*/
self_.runQuery = function (optOb,optParams) {
var options = getOptions_();
var de = new DataStoreEntity (kind_,self_);
// create query
var qo = self_.flatten(optOb,true);
var queryOb = de.querify(qo,optParams);
options.payload = JSON.stringify(queryOb);
var result= execute_ (self_.getEndpoint("runQuery"), options);
if (result.handleCode >=0) {
result.data = self_.unflatten (self_.reconstructProperties(result.result));
}
return result;
};
/**
* lookup an object by id
* @param {string} id - an id to lookup
* @return {object} the datastore result object and success codes
*/
self_.lookup = function (id) {
var options = getOptions_();
options.payload = JSON.stringify(new DataStoreEntity (kind_).lookupify(id) );
var result= execute_ (self_.getEndpoint("lookup"), options);
if (result.handleCode >=0) {
result.data = self_.unflatten (self_.reconstructProperties(result.result));
}
return result;
};
/**
* get the api url
* @param {string} method - the method
* @return {string} the url
*/
self_.getEndpoint = function (method) {
return 'https://www.googleapis.com/datastore/v1beta2/datasets/' + name_ + "/" + method;
};
/**
* get an array of ids
* @param {object} result the datastore result object and success codes
* @return {Array.string} the array of IDS
*/
self_.getIds = function (result) {
if (result.result.mutationResult ) {
return result.result.mutationResult.insertAutoIdKeys.map(function(d) { return d.path[0].id; }).filter(function(d) { return d });
}
else if (result.result.batch) {
return result.result.batch.entityResults.map(function(d) { return d.entity.key.path[0].id; }).filter(function(d) { return d });
}
else if (result.result.found) {
return result.result.found.map(function(d) { return d.entity.key.path[0].id; }).filter(function(d) { return d });
}
};
/**
* get basic http options
* @return {object} the options
*/
function getOptions_ () {
return {
method: "POST",
contentType : "application/json" ,
muteHttpExceptions : true,
headers: {
authorization: "Bearer " + accessToken_
}
};
}
/**
* exponetial backoff get
*/
function fetch_ (url,options) {
return parentHandler_.rateLimitExpBackoff ( function () {
return UrlFetchApp.fetch(url,options); }) ;
}
/**
* execute a fetch
*/
function execute_ (url,options) {
var result=null, error = '', code = enums.CODE.OK;
return parentHandler.rateLimitExpBackoff ( function () {
try {
var response = fetch_(url, options);
result = JSON.parse(response.getContentText());
if (response.getResponseCode() !== 200) {
code = enums.CODE.HTTP;
error = 'status'+ response.getResponseCode();
}
}
catch (err) {
code= enums.CODE.DRIVER;
error = err;
}
return {handleError:error, handleCode:code, result:result};
});
}
return self_;
}
/**
* entity for data store - takes care of organizing properties and keys
* @param {string} kind - somewhat like the table name
* @param {DataStoreWorker} worker to get handler from
* @return {DataStoreEntity} self
* @class DataStoreEntity
*/
function DataStoreEntity (kind,worker) {
var self_ = this;
var kind_ = kind;
var worker_ = worker;
var properties_ = {};
/*
* clear out current properties and optionall set some more
* @param {object} optOb the object to set if required
* @return {DataStoreEntity} self
*/
self_.resetProperties = function (optOb) {
properties_ = {};
if (optOb) {
self_.setProperties(optOb);
}
return self_;
};
/*
* add the object ob to the datastore properties
* @param {object} optOb the object to set
* @return {DataStoreEntity} self
*/
self_.setProperties = function (optOb) {
ob = optOb || {};
Object.keys(ob).forEach (function(k) {
properties_[k] = {};
properties_[k][getType_(ob[k])] = ob[k];
});
return self_;
};
/*
* get a single property value by property name
* @param {string} propertyName the property name
* @return {*} the value of the property
*/
self_.getProperty = function (propertyName) {
var p = properties_[propertyName] ;
// just takes the first data type
return p ? convertType(p,Object.keys(p)[0]) : null;
// strangely, it returns integer type as a string
function convertType (ob,type) {
if (type === "integerValue") {
return parseInt( p[type],10);
}
else {
return p[type];
}
}
};
/*
* get a single property type by property name
* @param {string} propertyName the property name
* @return {string} the datastore type of the property
*/
self_.getPropertyType = function (k) {
var p = properties_[k] ;
return p ? Object.keys(p)[0] : '';
};
/*
* get the datastore properties
* @return {Array.object} the properties
*/
self_.getProperties = function () {
return properties_;
};
/*
* introduce new datastore properties
* @return {DatastoreEntity} self
*/
self_.injectProperties = function(p) {
properties_ = p || {};
return self_;
};
/*
* recsontruct the datastore properties into a single object
* @return {object} the properties
*/
self_.reconstructProperties = function () {
return Object.keys(properties_).reduce ( function (p,c) {
p = self_.getProperty(c);
return p;
},{});
};
/*
* construct the key part of the object for looking up a single id
* @param {string} id the id
* @return {object} the keys
*/
self_.lookupify = function (id) {
return {keys: [ {path: [ { kind: kind_, id: id }] } ]};
};
/*
* make a stringifiable representation of an object ready for writing to the datastore
* @return {object} a constructed entity
*/
self_.objectify = function () {
return {key: {path: [ { kind: kind_ }] },properties:properties_};
};
/*
* make a stringifiable representation of an object ready for updating
* @param {string} id the id
* @return {object} a constructed entity
*/
self_.updateify = function (id) {
return {key: {path: [ { kind: kind_, id:id }] },properties:properties_};
};
/*
* make a stringifiable representation of an object ready for deleting
* @param {Array.string} ids array of ids to remove
* @return {object} a constructed entity
*/
self_.removeify = function (ids) {
if (!Array.isArray(ids)) ids = [ids];
return ids.map(function(d) { return {path: [{ "kind": kind_, "id": d }]}; });
};
/*
* make a stringifiable representation of an object ready for querying
* @param {Object} optQuery any sorting/skipping type params
* @param {Object} optParams any sroting/skipping type params
* @return {string} a datastore type name
*/
self_.querify = function (optQuery, optParams) {
var enums = worker_.getParentHandler().getEnums();
// the basic rule is
// all queries that are = can be combined
// you can only have multiple constraints (eg > & <) if the property is the same one
var q = {query:{ kinds: [{name: kind_}]}};
if (optParams) {
Object.keys(optParams).forEach (function(k){
q.query[k] = optParams[k];
});
}
if (optQuery) {
var ks = '';
// do the non-constraints first
var fs = Object.keys(optQuery).reduce (function(p,c) {
if (!optQuery.hasOwnProperty (enums.SETTINGS.CONSTRAINT)) {
p.push(patchOb_ (c,optQuery , enums.DATASTORE_CONSTRAINTS.EQ));
ks = c;
}
return p;
},[]);
var fs = Object.keys(optQuery).reduce (function(p,c) {
if (isObject_ (optQuery) ){
if (optQuery.hasOwnProperty (enums.SETTINGS.CONSTRAINT)) {
if (!ks || ks === c) {
optQuery[enums.SETTINGS.CONSTRAINT].forEach(function(d) {
var operator = worker_.getConstraintName(d.constraint);
if (operator) {
p.push(patchOb_ (c ,d.value, operator));
}
});
ks = c;
}
}
else {
throw 'unexpected unflattened property ' + JSON.stringify(optQuery);
}
}
return p;
},fs);
if (fs && fs.length > 0) {
q.query.filter = {
compositeFilter:{
operator: "AND",
filters: fs,
}
};
}
}
function patchOb_(name, v,oper) {
var o = { propertyFilter:
{ property: { name: name },
operator: oper,
value:{}
}
};
o.propertyFilter.value[getType_ (v)] = v;
return o;
}
return q;
};
function isObject_ (obj) {
return obj === Object(obj);
}
/*
* return a datastore type deduced from the value
* @param {*} a value
* @return {string} a datastore type name
*/
function getType_ (value) {
var t = typeof value;
if (t === "string") {
return "stringValue"
}
else if (t==="number") {
if (!isNaN(value) && parseInt(Number(value),10) === value) {
return "integerValue";
}
else {
return "doubleValue";
}
}
else if (t==="boolean") {
return "booleanValue"
}
else if (t==="undefined") {
return "stringValue"
}
else if (value instanceof Date) {
return "dateTimeValue"
}
else {
throw 'unable to determine type of ' + value;
}
}
return self_;
}
See more like this in Database abstraction with google apps script and Datastore driver