redis transactions hard prime

Author: Brian Detering

README.md

@@ -1,15 +1,15 @@
 # Redis Dataloader
 
 Batching and Caching layer using Redis as the Caching layer.
-Redis Dataloader is based on the [Facebook Dataloader](https://github.com/facebook/dataloader),
-and uses it internally.
+Redis Dataloader wraps [Facebook Dataloader](https://github.com/facebook/dataloader),
+adding Redis as a caching layer.
 
 ## Example
 
 ```javascript
-const redis = require('redis').createClient();
+const redisClient = require('redis').createClient();
 const DataLoader = require('dataloader');
-const RedisDataLoader = require('redis-dataloader')({ redis: redis });
+const RedisDataLoader = require('redis-dataloader')({ redis: redisClient });
 
 const loader = new RedisDataLoader(
     // set a prefix for the keys stored in redis. This way you can avoid key
@@ -20,11 +20,16 @@ const loader = new RedisDataLoader(
     // The options here are the same as the regular dataloader options, with
     // the additional option "expire"
     {
-        // caching here is a local in memory cache
+        // caching here is a local in memory cache. Caching is always done
+        // to redis.
         cache: true,
         // if set redis keys will be set to expire after this many seconds
         // this may be useful as a fallback for a redis cache.
-        expire: 60
+        expire: 60,
+        // can include a custom serialization and deserialization for
+        // storage in redis.
+        serialize: date => date.getTime(),
+        deserialize: timestamp => new Date(timestamp)
     }
 );
 
@@ -40,9 +45,10 @@ loader.clear(5).then(() => {})
 In general, RedisDataLoader has the same API as the Facebook Dataloader Api,
 with a few differences. Read through the [Facebook Dataloader documentation](https://github.com/facebook/dataloader) and then note the differences mentioned here.
 
-- `clear` returns a promise (waits until redis succeeds at deleting the key)
+- `clear` returns a promise (waits until redis succeeds at deleting the key). Facebook Dataloader's `clear` method is synchronous.
 - `clearAll` is not available (redis does not have an efficient way to do this?)
-- `prime` will always overwrite the redis cache. It in turn calls prime on the local cache (which does not adjust the cache if the key already exists)
+- `prime` will always overwrite the cache. Facebook Dataloader will only write to
+its cache if a value is not already present. Prime is asyncronous and returns a Promise.
 - dataloader results must be either `null` or a JSON object.
 
 ### Instantiation

index.js

@@ -7,19 +7,30 @@ const DataLoader = require('dataloader');
 module.exports = fig => {
     const redis = fig.redis;
 
-    const parse = resp => Q.Promise((resolve, reject) => {
+    const parse = (resp, opt) => Q.Promise((resolve, reject) => {
         try {
-            resolve(resp !== '' && resp !== null ? JSON.parse(resp) : resp);
+            if(resp === '' || resp === null) {
+                resolve(resp);
+            }
+            else if(opt.deserialize) {
+                resolve(opt.deserialize(resp));
+            }
+            else {
+                resolve(JSON.parse(resp));
+            }
         }
         catch(err) {
             reject(err);
         }
     });
 
-    const toString = val => {
+    const toString = (val, opt) => {
         if(val === null) {
             return Q('');
         }
+        else if(opt.serialize) {
+            return Q(opt.serialize(val));
+        }
         else if(_.isObject(val)) {
             return Q(JSON.stringify(val));
         }
@@ -28,42 +39,49 @@ module.exports = fig => {
         }
     };
 
-    const rSet = (keySpace, key, rawVal, expire) => toString(rawVal)
-    .then(val => Q.Promise((resolve, reject) => redis.set(
-        `${keySpace}:${key}`, val, (err, resp) => {
-            if(err) {
-                reject(err);
-            }
-            else {
-                if(expire) {
-                    redis.expire(`${keySpace}:${key}`, expire);
-                }
-                resolve(resp);
-            }
+    const makeKey = (keySpace, key) => `${keySpace}:${key}`;
+
+    const rSetAndGet = (keySpace, key, rawVal, opt) => toString(rawVal, opt)
+    .then(val => Q.Promise((resolve, reject) => {
+        const fullKey = makeKey(keySpace, key);
+        const multi = redis.multi();
+        multi.set(fullKey, val);
+        if(opt.expire) {
+            multi.expire(fullKey, opt.expire);
         }
-    )));
+        multi.get(fullKey);
+        multi.exec((err, replies) => err ?
+            reject(err) : parse(_.last(replies), opt).then(resolve)
+        );
+    }));
+
+    const rGet = (keySpace, key, opt) => Q.Promise(
+        (resolve, reject) => redis.get(
+            makeKey(keySpace, key),
+            (err, result) => err ? reject(err) : parse(result, opt).then(resolve)
+        )
+    );
 
-    const rMGet = (keySpace, keys) => {
-        return Q.Promise((resolve, reject) => redis.mget(
-            _.map(keys, k => `${keySpace}:${k}`),
+    const rMGet = (keySpace, keys, opt) => Q.Promise(
+        (resolve, reject) => redis.mget(
+            _.map(keys, k => makeKey(keySpace, k)),
             (err, results) => err ?
                 reject(err) :
-                Q.all(_.map(results, parse)).then(resolve)
-        ));
-    };
+                Q.all(_.map(results, r => parse(r, opt))).then(resolve)
+        )
+    );
 
     const rDel = (keySpace, key) => Q.Promise((resolve, reject) => redis.del(
-        `${keySpace}:${key}`, (err, resp) => err ? reject(err) : resolve(resp)
+        makeKey(keySpace, key), (err, resp) => err ? reject(err) : resolve(resp)
     ));
 
     return class RedisDataLoader {
-        constructor(ks, userLoader, options) {
+        constructor(ks, userLoader, opt) {
+            const customOptions = ['expire', 'serialize', 'deserialize'];
+            this.opt = _.pick(opt, customOptions) || {};
             this.keySpace = ks;
-
-            this.expire = options && options.expire;
-
             this.loader = new DataLoader(
-                keys => rMGet(this.keySpace, keys)
+                keys => rMGet(this.keySpace, keys, this.opt)
                 .then(results => Q.all(_.map(
                     results,
                     (v, i) => {
@@ -72,17 +90,17 @@ module.exports = fig => {
                         }
                         else if(v === null) {
                             return userLoader.load(keys[i])
-                            .then(resp => {
-                                return rSet(this.keySpace, keys[i], resp, this.expire)
-                                .then(() => resp);
-                            });
+                            .then(resp => rSetAndGet(
+                                this.keySpace, keys[i], resp, this.opt
+                            ))
+                            .then(r => r === '' ? null : r);
                         }
                         else {
                             return Q(v);
                         }
                     }
                 ))),
-                _.omit(options, 'expire')
+                _.omit(opt, customOptions)
             );
         }
 
@@ -95,8 +113,8 @@ module.exports = fig => {
         }
 
         prime(key, val) {
-            return rSet(this.keySpace, key, val, this.expire)
-            .then(() => this.loader.prime(key, val));
+            return rSetAndGet(this.keySpace, key, val, this.opt)
+            .then(resp => this.loader.clear(key).prime(key, resp));
         }
 
         clear(key) {

package.json

@@ -1,6 +1,6 @@
 {
   "name": "redis-dataloader",
-  "version": "0.1.0",
+  "version": "0.2.0",
   "description": "DataLoader Using Redis as a Cache",
   "main": "index.js",
   "scripts": {
@@ -14,7 +14,8 @@
     "DataLoader",
     "Cache",
     "Redis",
-    "Batch"
+    "Batch",
+    "Facebook"
   ],
   "author": "Brian Detering",
   "license": "MIT",

test.js

@@ -50,27 +50,6 @@ describe('redis-dataloader', () => {
     afterEach(() => _.each(this.stubs, s => s.restore()));
 
     describe('load', () => {
-        it('should handle redis key expiration if set', done => {
-            const loader = new RedisDataLoader(
-                this.keySpace,
-                this.userLoader(),
-                { cache: false, expire: 1 }
-            );
-
-            loader.load('json')
-            .then(data => {
-                expect(data).to.deep.equal(this.data.json);
-                setTimeout(() => {
-                    loader.load('json')
-                    .then(data => {
-                        expect(data).to.deep.equal(this.data.json);
-                        expect(this.loadFn.callCount).to.equal(2);
-                        done();
-                    }).done();
-                }, 1100);
-            }).done();
-        });
-
         it('should load json value', done => {
             this.loader.load('json').then(data => {
                 expect(data).to.deep.equal(this.data.json);
@@ -137,6 +116,45 @@ describe('redis-dataloader', () => {
                 done();
             }).done();
         });
+
+        it('should handle redis key expiration if set', done => {
+            const loader = new RedisDataLoader(
+                this.keySpace,
+                this.userLoader(),
+                { cache: false, expire: 1 }
+            );
+
+            loader.load('json')
+            .then(data => {
+                expect(data).to.deep.equal(this.data.json);
+                setTimeout(() => {
+                    loader.load('json')
+                    .then(data => {
+                        expect(data).to.deep.equal(this.data.json);
+                        expect(this.loadFn.callCount).to.equal(2);
+                        done();
+                    }).done();
+                }, 1100);
+            }).done();
+        });
+
+        it('should handle custom serialize and deserialize method', done => {
+            const loader = new RedisDataLoader(
+                this.keySpace,
+                this.userLoader(),
+                {
+                    serialize: v => 100,
+                    deserialize: v => new Date(Number(v))
+                }
+            );
+
+            loader.load('json')
+            .then(data => {
+                expect(data).to.be.instanceof(Date);
+                expect(data.getTime()).to.equal(100);
+                done();
+            }).done();
+        });
     });
 
     describe('loadMany', () => {
@@ -182,7 +200,8 @@ describe('redis-dataloader', () => {
         });
 
         it('should require a key', done => {
-            this.loader.clear().catch(err => {
+            this.loader.clear()
+            .catch(err => {
                 expect(err.message).to.equal('Key parameter is required');
                 done();
             }).done();