From 305e7a6f627ead8fdb3ee7d28bc2159081c2fea4 Mon Sep 17 00:00:00 2001 From: Pedro Ferreira Date: Fri, 12 Aug 2016 11:33:59 +0200 Subject: [PATCH] LiveSync: fixed unit tests --- livesync/tests/agent_test.py | 15 ++++--- livesync/tests/simplify_test.py | 41 +++++++++++++------ livesync/tests/uploader_test.py | 70 ++++++++++++++++++++------------- livesync/tests/util_test.py | 70 ++------------------------------- 4 files changed, 81 insertions(+), 115 deletions(-) diff --git a/livesync/tests/agent_test.py b/livesync/tests/agent_test.py index 4f075dc..b0e7b7b 100644 --- a/livesync/tests/agent_test.py +++ b/livesync/tests/agent_test.py @@ -17,7 +17,6 @@ from mock import MagicMock from indico_livesync.base import LiveSyncBackendBase -from indico_livesync.models.agents import LiveSyncAgent from indico_livesync.models.queue import LiveSyncQueueEntry, ChangeType, EntryType @@ -61,13 +60,13 @@ def test_run(mocker): assert mock_uploader.run.called -def test_fetch_records(db, dummy_event_new): +def test_fetch_records(db, dummy_event_new, dummy_agent): """Test if the correct records are fetched""" - agent = LiveSyncAgent(backend_name='dummy', name='dummy') - backend = DummyBackend(agent) - db.session.add(agent) - queue = [LiveSyncQueueEntry(change=ChangeType.created, type=EntryType.event, event=dummy_event_new, processed=True), - LiveSyncQueueEntry(change=ChangeType.created, type=EntryType.event, event=dummy_event_new)] - agent.queue = queue + backend = DummyBackend(dummy_agent) + queue = [ + LiveSyncQueueEntry(change=ChangeType.created, type=EntryType.event, event_new=dummy_event_new, processed=True), + LiveSyncQueueEntry(change=ChangeType.created, type=EntryType.event, event_new=dummy_event_new) + ] + dummy_agent.queue = queue db.session.flush() assert backend.fetch_records() == [queue[1]] diff --git a/livesync/tests/simplify_test.py b/livesync/tests/simplify_test.py index 51263f1..67a6db2 100644 --- a/livesync/tests/simplify_test.py +++ b/livesync/tests/simplify_test.py @@ -21,6 +21,10 @@ from indico_livesync import process_records, SimpleChange from indico_livesync.models.queue import LiveSyncQueueEntry, ChangeType, EntryType +class Dummy(object): + pass + + @pytest.fixture def queue_entry_dummy_object(monkeypatch): monkeypatch.setattr(LiveSyncQueueEntry, 'object', object) @@ -36,7 +40,7 @@ def queue_entry_dummy_object(monkeypatch): @pytest.mark.usefixtures('queue_entry_dummy_object') def test_process_records_category_ignored(mocker, change, invalid): """Test if categories are only kepy for certain changes""" - cascade = mocker.patch('indico_livesync.simplify._cascade') + cascade = mocker.patch('indico_livesync.simplify._process_cascaded') cascade.return_value = [object()] records = [LiveSyncQueueEntry(change=change, type=EntryType.category)] if invalid: @@ -58,38 +62,49 @@ def test_process_records_category_ignored(mocker, change, invalid): @pytest.mark.usefixtures('queue_entry_dummy_object') def test_process_records_cascade(mocker, change, cascade): """Test if certain changes cascade to child elements""" - cascade_mock = mocker.patch('indico_livesync.simplify._cascade') + cascade_mock = mocker.patch('indico_livesync.simplify._process_cascaded') records = [LiveSyncQueueEntry(change=change)] process_records(records) - assert cascade_mock.called == cascade + assert cascade_mock.call_args == (({records[0]} if cascade else set(),),) @pytest.mark.parametrize('changes', bool_matrix('......')) -@pytest.mark.usefixtures('queue_entry_dummy_object') -def test_process_records_simplify(changes): +def test_process_records_simplify(changes, mocker, db, create_event, dummy_agent): """Test if queue entries for the same object are properly simplified""" + event1 = create_event(id_=1) + event2 = create_event(id_=2) + + db.session.add(dummy_agent) + db.session.add(event1) + db.session.add(event2) + refs = ( - LiveSyncQueueEntry(type=EntryType.event, event_id=1).object_ref, - LiveSyncQueueEntry(type=EntryType.event, event_id=2).object_ref + {'type': EntryType.event, 'event_id': event1.id}, + {'type': EntryType.event, 'event_id': event2.id} ) + queue = [] changes = changes[:3], changes[3:] expected = [0, 0] for i, ref in enumerate(refs): if changes[i][0]: - queue.append(LiveSyncQueueEntry(change=ChangeType.created, **ref)) + queue.append(LiveSyncQueueEntry(change=ChangeType.created, agent=dummy_agent, **ref)) expected[i] |= SimpleChange.created if changes[i][1]: - queue.append(LiveSyncQueueEntry(change=ChangeType.data_changed, **ref)) - queue.append(LiveSyncQueueEntry(change=ChangeType.data_changed, **ref)) + queue += [LiveSyncQueueEntry(change=ChangeType.data_changed, agent=dummy_agent, **ref), + LiveSyncQueueEntry(change=ChangeType.data_changed, agent=dummy_agent, **ref)] expected[i] |= SimpleChange.updated if changes[i][2]: - queue.append(LiveSyncQueueEntry(change=ChangeType.deleted, **ref)) + queue.append(LiveSyncQueueEntry(change=ChangeType.deleted, agent=dummy_agent, **ref)) expected[i] |= SimpleChange.deleted + db.session.flush() + result = process_records(queue) assert result == process_records(reversed(queue)) # queue order shouldn't matter assert len(result) == sum(1 for x in expected if x) + + result_refs = {obj.id: change for obj, change in result.viewitems()} for i, ref in enumerate(refs): - assert (ref in result) == bool(expected[i]) - assert result[ref] == expected[i] + assert (ref['event_id'] in list(result_refs)) == bool(expected[i]) + assert result_refs.get(ref['event_id'], 0) == expected[i] diff --git a/livesync/tests/uploader_test.py b/livesync/tests/uploader_test.py index a6a45a8..c76b91b 100644 --- a/livesync/tests/uploader_test.py +++ b/livesync/tests/uploader_test.py @@ -14,9 +14,7 @@ # You should have received a copy of the GNU General Public License # along with Indico; if not, see . -import pytest from mock import MagicMock, Mock -from werkzeug.datastructures import ImmutableDict from indico_livesync import SimpleChange from indico_livesync.models.queue import LiveSyncQueueEntry, ChangeType, EntryType @@ -25,10 +23,6 @@ from indico_livesync.uploader import Uploader, MARCXMLUploader from MaKaC.conference import Conference -def _rm_none(dict_): - return ImmutableDict((k, v) for k, v in dict_.iteritems() if v is not None) - - class RecordingUploader(Uploader): """An uploader which logs each 'upload'""" def __init__(self, *args, **kwargs): @@ -38,7 +32,8 @@ class RecordingUploader(Uploader): def upload_records(self, records, from_queue): if from_queue: - self._uploaded.append((set((_rm_none(rec), op) for rec, op in records.iteritems()), from_queue)) + recs = set(records.viewitems()) + self._uploaded.append((recs, from_queue)) else: self._uploaded.append((set(records), from_queue)) @@ -60,11 +55,6 @@ class FailingUploader(RecordingUploader): raise Exception('All your data are belong to us!') -@pytest.fixture(autouse=True) -def mock_resolved_zodb_objects(mocker): - mocker.patch.object(LiveSyncQueueEntry, 'object', autospec=True) - - def test_run_initial(mocker): """Test the initial upload""" mocker.patch.object(Uploader, 'processed_records', autospec=True) @@ -79,49 +69,73 @@ def test_run_initial(mocker): assert not uploader.processed_records.called -def test_run(mocker): +def test_run(mocker, db, create_event, dummy_agent): """Test uploading queued data""" - db = mocker.patch('indico_livesync.uploader.db') uploader = RecordingUploader(MagicMock()) uploader.BATCH_SIZE = 3 - records = tuple(LiveSyncQueueEntry(change=ChangeType.created, type=EntryType.event, event_id=evt_id) - for evt_id in xrange(4)) + + events = tuple(create_event(id_=evt_id) for evt_id in xrange(4)) + records = tuple(LiveSyncQueueEntry(change=ChangeType.created, type=EntryType.event, event_id=evt.id, + agent=dummy_agent) + for evt in events) + for rec in records: + db.session.add(rec) + db.session.flush() + + db_mock = mocker.patch('indico_livesync.uploader.db') + uploader.run(records) - refs = tuple((_rm_none(record.object_ref), int(SimpleChange.created)) for record in records) - batches = set(refs[:3]), set(refs[3:]) + + objs = tuple((record.object, int(SimpleChange.created)) for record in records) + batches = set(objs[:3]), set(objs[3:]) assert uploader.all_uploaded == [(batches[0], True), (batches[1], True)] # All records should be marked as processed assert all(record.processed for record in records) # Marking records as processed is committed immediately - assert db.session.commit.call_count == 2 + assert db_mock.session.commit.call_count == 2 -def test_run_failing(mocker): +def test_run_failing(mocker, db, create_event, dummy_agent): """Test a failing queue run""" - db = mocker.patch('indico_livesync.uploader.db') uploader = FailingUploader(MagicMock()) uploader.BATCH_SIZE = 3 - records = tuple(LiveSyncQueueEntry(change=ChangeType.created, type=EntryType.event, event_id=evt_id) - for evt_id in xrange(10)) + + events = tuple(create_event(id_=evt_id) for evt_id in xrange(10)) + records = tuple(LiveSyncQueueEntry(change=ChangeType.created, type=EntryType.event, event_id=evt.id, + agent=dummy_agent) + for evt in events) + + for rec in records: + db.session.add(rec) + db.session.flush() + + db_mock = mocker.patch('indico_livesync.uploader.db') + uploader.run(records) - refs = tuple((_rm_none(record.object_ref), int(SimpleChange.created)) for record in records) + objs = tuple((record.object, int(SimpleChange.created)) for record in records) assert uploader.logger.exception.called # No uploads should happen after a failed batch - assert uploader._uploaded == [(set(refs[:3]), True), (set(refs[3:6]), True)] + assert uploader._uploaded == [(set(objs[:3]), True), (set(objs[3:6]), True)] # Only successful records should be marked as processed assert all(record.processed for record in records[:3]) assert not any(record.processed for record in records[3:]) # Only the first uccessful batch should have triggered a commit - assert db.session.commit.call_count == 1 + assert db_mock.session.commit.call_count == 1 -def test_marcxml_run(mocker): +def test_marcxml_run(mocker, db, dummy_event_new, dummy_agent): """Text if the MARCXML uploader uses the correct function""" mocker.patch('indico_livesync.uploader.db') mocker.patch.object(MARCXMLUploader, 'upload_xml', autospec=True) mxg = mocker.patch('indico_livesync.uploader.MARCXMLGenerator') + + entry = LiveSyncQueueEntry(change=ChangeType.created, type=EntryType.event, event_new=dummy_event_new, + agent=dummy_agent) + db.session.add(entry) + db.session.flush() + uploader = MARCXMLUploader(MagicMock()) - uploader.run([LiveSyncQueueEntry(change=ChangeType.created)]) + uploader.run([entry]) assert mxg.records_to_xml.called assert not mxg.objects_to_xml.called assert uploader.upload_xml.called diff --git a/livesync/tests/util_test.py b/livesync/tests/util_test.py index f976118..98b6f4b 100644 --- a/livesync/tests/util_test.py +++ b/livesync/tests/util_test.py @@ -16,73 +16,19 @@ from datetime import timedelta -import pytest - -from indico.modules.events.contributions import Contribution -from indico.modules.events.contributions.models.subcontributions import SubContribution from indico.util.date_time import now_utc -from indico_livesync.models.agents import LiveSyncAgent from indico_livesync.models.queue import LiveSyncQueueEntry, ChangeType, EntryType from indico_livesync.plugin import LiveSyncPlugin -from indico_livesync.util import make_compound_id, clean_old_entries, get_excluded_categories +from indico_livesync.util import clean_old_entries -class MockCategory(object): - def __init__(self, id_, subcategories=None): - self.id = id_ - self.subcategories = subcategories or set() - - def getId(self): - return self.id - - -class MockCategoryManager(object): - # a - # |- b - # `- c - # |- d - # |- e - # `- f - categories = { - 'a': MockCategory('a', {'b', 'c'}), - 'b': MockCategory('b'), - 'c': MockCategory('c', {'d'}), - 'd': MockCategory('d', {'e', 'f'}), - 'e': MockCategory('e'), - 'f': MockCategory('f') - } - - @classmethod - def getById(cls, id_): - return cls.categories[id_] - - -@pytest.mark.parametrize(('ref', 'expected'), ( - ({'type': EntryType.event, 'event_id': '123'}, '123'), - ({'type': EntryType.contribution, 'contrib_id': '456'}, '123.456'), - ({'type': EntryType.subcontribution, 'subcontrib_id': '789'}, '123.456.789'), -)) -def test_make_compound_id(create_event, ref, expected): - evt = create_event(123) - evt.contributions = [Contribution(id=456, title='test', duration=timedelta(hours=1))] - evt.contributions[0].subcontributions = [SubContribution(id=789, title='foo', duration=timedelta(minutes=10))] - assert make_compound_id(ref) == expected - - -@pytest.mark.parametrize('ref_type', ('unknown', 'category')) -def test_make_compound_id_errors(ref_type): - with pytest.raises(ValueError): - make_compound_id({'type': ref_type}) - - -def test_clean_old_entries(dummy_event_new, db): +def test_clean_old_entries(dummy_event_new, db, dummy_agent): now = now_utc() - agent = LiveSyncAgent(name='dummy', backend_name='dummy') for processed in (True, False): for day in range(10): - db.session.add(LiveSyncQueueEntry(agent=agent, change=ChangeType.created, type=EntryType.event, - event=dummy_event_new, processed=processed, + db.session.add(LiveSyncQueueEntry(agent=dummy_agent, change=ChangeType.created, type=EntryType.event, + event_new=dummy_event_new, processed=processed, timestamp=now - timedelta(days=day, hours=12))) db.session.flush() # Nothing deleted with the setting's default value @@ -97,11 +43,3 @@ def test_clean_old_entries(dummy_event_new, db): clean_old_entries() assert LiveSyncQueueEntry.find(processed=False).count() == 10 assert LiveSyncQueueEntry.find(processed=True).count() == 3 - - -def test_excluded_categories(mocker, monkeypatch): - """Test if category exclusions work""" - monkeypatch.setattr('indico_livesync.util.CategoryManager', MockCategoryManager) - plugin = mocker.patch('indico_livesync.plugin.LiveSyncPlugin') - plugin.settings.get.return_value = [{'id': 'invalid'}, {'id': 'c'}, {'id': 'd'}] - assert get_excluded_categories() == {'c', 'd', 'e', 'f'}