From df15795ae48aa33bec734779d934887b5acda994 Mon Sep 17 00:00:00 2001 From: rlloyd Date: Tue, 9 Oct 2018 04:50:24 -0400 Subject: [PATCH 01/18] start a requirements.txt --- .idea/vcs.xml | 6 ++++++ requirements.txt | 1 + 2 files changed, 7 insertions(+) create mode 100644 .idea/vcs.xml create mode 100644 requirements.txt diff --git a/.idea/vcs.xml b/.idea/vcs.xml new file mode 100644 index 0000000..94a25f7 --- /dev/null +++ b/.idea/vcs.xml @@ -0,0 +1,6 @@ + + + + + + \ No newline at end of file diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..a3498aa --- /dev/null +++ b/requirements.txt @@ -0,0 +1 @@ +pymongo==3.7.1 From 1f0845b62e15295acbcb17dd1b044faab907e004 Mon Sep 17 00:00:00 2001 From: rlloyd Date: Tue, 9 Oct 2018 05:22:19 -0400 Subject: [PATCH 02/18] Update to use pymongo.MongoClient --- mongoqueue/test.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/mongoqueue/test.py b/mongoqueue/test.py index 5128d9a..4b420d3 100644 --- a/mongoqueue/test.py +++ b/mongoqueue/test.py @@ -14,7 +14,7 @@ class MongoLockTest(TestCase): def setUp(self): - self.client = pymongo.Connection(os.environ.get("TEST_MONGODB")) + self.client = pymongo.MongoClient(os.environ.get("TEST_MONGODB")) self.db = self.client.test_queue self.collection = self.db.locks @@ -54,7 +54,7 @@ def test_auto_expires_old(self): class MongoQueueTest(TestCase): def setUp(self): - self.client = pymongo.Connection(os.environ.get("TEST_MONGODB")) + self.client = pymongo.MongoClient(os.environ.get("TEST_MONGODB")) self.db = self.client.test_queue self.queue = MongoQueue(self.db.queue_1, "consumer_1") @@ -145,7 +145,7 @@ def test_context_manager_error(self): pass job = self.queue.next() - self.assertEqual(job.data['attempts'], 1) + self.assertEqual(job.attempts, 1) def test_context_manager_complete(self): self.queue.put({"foobar": 1}) From f135c337ede6b9d3ba5923e8b2e6786e7f98e3ca Mon Sep 17 00:00:00 2001 From: rlloyd Date: Tue, 9 Oct 2018 05:23:14 -0400 Subject: [PATCH 03/18] update uuid usage --- mongoqueue/lock.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/mongoqueue/lock.py b/mongoqueue/lock.py index 9d7732d..a8070c0 100644 --- a/mongoqueue/lock.py +++ b/mongoqueue/lock.py @@ -22,7 +22,7 @@ class MongoLock(object): def __init__(self, collection, lock_name, lease=120): self.collection = collection self.lock_name = lock_name - self._client_id = uuid.uuid4().get_hex() + self._client_id = uuid.uuid4().hex self._locked = False self._lease_time = lease self._lock_expires = False @@ -57,7 +57,7 @@ def _acquire(self): '_id': self.lock_name, 'ttl': ttl, 'client_id': self._client_id}, - w=1, j=1) + w=1, j=True) except errors.DuplicateKeyError: self.collection.remove( {"_id": self.lock_name, 'ttl': {'$lt': datetime.now()}}) @@ -65,7 +65,7 @@ def _acquire(self): self.collection.insert( {'_id': self.lock_name, 'ttl': ttl, - 'client_id': self._client_id}, w=1, j=1) + 'client_id': self._client_id}, w=1, j=True) except errors.DuplicateKeyError: self._locked = False return self._locked From 3d919edc533c1f9e9059c57d277fafe060a95957 Mon Sep 17 00:00:00 2001 From: rlloyd Date: Tue, 9 Oct 2018 05:45:03 -0400 Subject: [PATCH 04/18] remove deprecated code from unit tests --- mongoqueue/test.py | 18 +----------------- 1 file changed, 1 insertion(+), 17 deletions(-) diff --git a/mongoqueue/test.py b/mongoqueue/test.py index 4b420d3..dece914 100644 --- a/mongoqueue/test.py +++ b/mongoqueue/test.py @@ -24,7 +24,7 @@ def tearDown(self): def test_lock_acquire_release_context_manager(self): with lock(self.collection, 'test1') as l: self.assertTrue(l.locked) - self.assertEqual(self.collection.find().count(), 0) + self.assertEqual(self.collection.count_documents(filter={}), 0) def test_auto_expires_old(self): lock = MongoLock(self.collection, 'test2', lease=2) @@ -117,22 +117,6 @@ def test_error(self): def test_progress(self): pass - def test_stats(self): - - for i in range(5): - data = {"context_id": "alpha", - "data": [1, 2, 3], - "more-data": time.time()} - self.queue.put(data) - job = self.queue.next() - job.error("problem") - - stats = self.queue.stats() - self.assertEqual({'available': 5, - 'total': 5, - 'locked': 0, - 'errors': 0}, stats) - def test_context_manager_error(self): self.queue.put({"foobar": 1}) job = self.queue.next() From 91236954e5a1417b80b4cdd6f4c35911c666161f Mon Sep 17 00:00:00 2001 From: rlloyd Date: Tue, 9 Oct 2018 06:00:25 -0400 Subject: [PATCH 05/18] add travis file --- .travis.yml | 10 ++++++++++ 1 file changed, 10 insertions(+) create mode 100644 .travis.yml diff --git a/.travis.yml b/.travis.yml new file mode 100644 index 0000000..0d23f09 --- /dev/null +++ b/.travis.yml @@ -0,0 +1,10 @@ +language: python +python: + - "3.6" + - "3.7" +# command to install dependencies +install: + - pip install -r requirements.txt +# command to run tests +script: + - python -m unittest discover \ No newline at end of file From becc62d60cc764d23f8b45d1b9dc6940dc5e74d1 Mon Sep 17 00:00:00 2001 From: rlloyd Date: Tue, 9 Oct 2018 06:07:55 -0400 Subject: [PATCH 06/18] add mongo service --- .travis.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 0d23f09..0814fa2 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,8 +1,10 @@ language: python python: - "3.6" - - "3.7" + - "3.7-dev" # command to install dependencies +services: + - mongodb install: - pip install -r requirements.txt # command to run tests From 55bbe0179b173de0dc4f65c4a2d7fb3ae07b7f0e Mon Sep 17 00:00:00 2001 From: rlloyd Date: Tue, 9 Oct 2018 06:10:33 -0400 Subject: [PATCH 07/18] try older pythons --- .travis.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.travis.yml b/.travis.yml index 0814fa2..335497b 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,5 +1,9 @@ language: python python: + - "2.6" + - "2.7" + - "3.4" + - "3.5" - "3.6" - "3.7-dev" # command to install dependencies From 499659276c1eace6f36a72d4dba03a617a3779b7 Mon Sep 17 00:00:00 2001 From: rlloyd Date: Tue, 9 Oct 2018 06:15:45 -0400 Subject: [PATCH 08/18] remove python 2.6 --- .travis.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 335497b..fe82467 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,6 +1,5 @@ language: python python: - - "2.6" - "2.7" - "3.4" - "3.5" From 6e06ca0ed0de4df8c9af4d69f3df6e0b9b29a0fb Mon Sep 17 00:00:00 2001 From: rlloyd Date: Tue, 9 Oct 2018 06:19:50 -0400 Subject: [PATCH 09/18] add git status --- README.rst | 3 +++ 1 file changed, 3 insertions(+) diff --git a/README.rst b/README.rst index af9bd4f..5cebb54 100644 --- a/README.rst +++ b/README.rst @@ -1,6 +1,9 @@ mongoqueue ---------- +.. image:: https://travis-ci.com/rupello/mongoqueue.svg?branch=master + :target: https://travis-ci.com/rupello/mongoqueue + Properties ========== From 17ead79d2b73099a76ee3f22e6715af99b5271a4 Mon Sep 17 00:00:00 2001 From: rlloyd Date: Tue, 9 Oct 2018 08:54:38 -0400 Subject: [PATCH 10/18] update readme --- README.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.rst b/README.rst index 5cebb54..6a988b4 100644 --- a/README.rst +++ b/README.rst @@ -26,10 +26,10 @@ A queue can be instantiated with a mongo collection and a consumer identifier. The consumer identifier helps distinguish multiple queue consumers that are taking jobs from the queue:: - >> from pymongo import Connection + >> from pymongo import MongoClient >> from mongoqueue import MongoQueue >> queue = MongoQueue( - ... Connection(TEST_DB).doctest_queue, + ... MongoClient(TEST_DB).doctest_queue, ... consumer_id="consumer-1", ... timeout=300, ... max_attempts=3) From f8284af20a396fc58e7b5d2ff544000ca75342fe Mon Sep 17 00:00:00 2001 From: rlloyd Date: Tue, 9 Oct 2018 09:37:49 -0400 Subject: [PATCH 11/18] remocve ide files --- .gitignore | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.gitignore b/.gitignore index d2d6f36..deead2d 100644 --- a/.gitignore +++ b/.gitignore @@ -33,3 +33,6 @@ nosetests.xml .mr.developer.cfg .project .pydevproject + +# PyCharm +.idea/ From bd73348bdd866ebc2d0843e6ca3b529eb3585a98 Mon Sep 17 00:00:00 2001 From: rlloyd Date: Tue, 9 Oct 2018 09:39:49 -0400 Subject: [PATCH 12/18] remove ide files --- .idea/vcs.xml | 6 ------ 1 file changed, 6 deletions(-) delete mode 100644 .idea/vcs.xml diff --git a/.idea/vcs.xml b/.idea/vcs.xml deleted file mode 100644 index 94a25f7..0000000 --- a/.idea/vcs.xml +++ /dev/null @@ -1,6 +0,0 @@ - - - - - - \ No newline at end of file From 9bc9fc4f1c59b47dc609b994a5ec0e5188a7f04d Mon Sep 17 00:00:00 2001 From: rlloyd Date: Tue, 9 Oct 2018 09:46:02 -0400 Subject: [PATCH 13/18] update requirements constraint --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index a3498aa..e8bdad3 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1 +1 @@ -pymongo==3.7.1 +pymongo>=3.7.0 From 6b1b096be666823ba0016bfd13c13f2ead8d8d73 Mon Sep 17 00:00:00 2001 From: rlloyd Date: Tue, 9 Oct 2018 09:57:04 -0400 Subject: [PATCH 14/18] add CI tests for multiple PyMOngo versions --- .travis.yml | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index fe82467..8b77f5f 100644 --- a/.travis.yml +++ b/.travis.yml @@ -5,11 +5,15 @@ python: - "3.5" - "3.6" - "3.7-dev" -# command to install dependencies +env: + - PYMONGO_VERSION=3.6.1 + - PYMONGO_VERSION=3.7.1 + - PYMONGO_VERSION=3.7.1 services: - mongodb +# command to install dependencies install: - - pip install -r requirements.txt + - pip install -q pymongo==$PYMONGO_VERSION # command to run tests script: - python -m unittest discover \ No newline at end of file From d2a637a901b0253e54f4afcc64af1835aa8c46fe Mon Sep 17 00:00:00 2001 From: rlloyd Date: Tue, 9 Oct 2018 10:24:43 -0400 Subject: [PATCH 15/18] remove count_documents as this breaks pymongo 3.6.1 --- mongoqueue/test.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mongoqueue/test.py b/mongoqueue/test.py index dece914..3ac7dbd 100644 --- a/mongoqueue/test.py +++ b/mongoqueue/test.py @@ -24,7 +24,7 @@ def tearDown(self): def test_lock_acquire_release_context_manager(self): with lock(self.collection, 'test1') as l: self.assertTrue(l.locked) - self.assertEqual(self.collection.count_documents(filter={}), 0) + self.assertEqual(self.collection.count(),0) def test_auto_expires_old(self): lock = MongoLock(self.collection, 'test2', lease=2) From a70410f87aa442f38487dbde1fda0e1d15840932 Mon Sep 17 00:00:00 2001 From: rlloyd Date: Tue, 9 Oct 2018 10:32:54 -0400 Subject: [PATCH 16/18] test older versions --- .travis.yml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index 8b77f5f..e62c00d 100644 --- a/.travis.yml +++ b/.travis.yml @@ -6,9 +6,10 @@ python: - "3.6" - "3.7-dev" env: - - PYMONGO_VERSION=3.6.1 - - PYMONGO_VERSION=3.7.1 - PYMONGO_VERSION=3.7.1 + - PYMONGO_VERSION=3.1 + - PYMONGO_VERSION=3.0 + - PYMONGO_VERSION=2.9.5 services: - mongodb # command to install dependencies From aa431e1f424acdc2764e2feb86a03d0c908fe7c0 Mon Sep 17 00:00:00 2001 From: rlloyd Date: Tue, 9 Oct 2018 10:51:00 -0400 Subject: [PATCH 17/18] PyMongo 2.0 --- .travis.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index e62c00d..d821fad 100644 --- a/.travis.yml +++ b/.travis.yml @@ -7,9 +7,8 @@ python: - "3.7-dev" env: - PYMONGO_VERSION=3.7.1 - - PYMONGO_VERSION=3.1 - PYMONGO_VERSION=3.0 - - PYMONGO_VERSION=2.9.5 + - PYMONGO_VERSION=2.0 services: - mongodb # command to install dependencies From cce4990273d750eaced60000a8811befe2f3d779 Mon Sep 17 00:00:00 2001 From: rlloyd Date: Tue, 9 Oct 2018 11:19:57 -0400 Subject: [PATCH 18/18] update pymongo lib versions to 2.4 onwards --- .travis.yml | 4 ++-- requirements.txt | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.travis.yml b/.travis.yml index d821fad..8dd46f0 100644 --- a/.travis.yml +++ b/.travis.yml @@ -6,9 +6,9 @@ python: - "3.6" - "3.7-dev" env: - - PYMONGO_VERSION=3.7.1 + - PYMONGO_VERSION=3.7 - PYMONGO_VERSION=3.0 - - PYMONGO_VERSION=2.0 + - PYMONGO_VERSION=2.4 services: - mongodb # command to install dependencies diff --git a/requirements.txt b/requirements.txt index e8bdad3..ffa4243 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1 +1 @@ -pymongo>=3.7.0 +pymongo>=2.4