From 776c484fb0bed5cd7056b889cb53e037faf248aa Mon Sep 17 00:00:00 2001 From: Pedro Ferreira Date: Fri, 6 Mar 2015 16:30:34 +0100 Subject: [PATCH] VC/Vidyo: Add Vidyo room clean-up task + CLI Added command to list Vidyo rooms, unit tests --- vc_vidyo/indico_vc_vidyo/cli.py | 71 +++++++++++++++++++++++ vc_vidyo/indico_vc_vidyo/plugin.py | 10 +++- vc_vidyo/indico_vc_vidyo/task.py | 82 +++++++++++++++++++++++++++ vc_vidyo/setup.py | 1 + vc_vidyo/tests/task_test.py | 91 ++++++++++++++++++++++++++++++ 5 files changed, 252 insertions(+), 3 deletions(-) create mode 100644 vc_vidyo/indico_vc_vidyo/cli.py create mode 100644 vc_vidyo/indico_vc_vidyo/task.py create mode 100644 vc_vidyo/tests/task_test.py diff --git a/vc_vidyo/indico_vc_vidyo/cli.py b/vc_vidyo/indico_vc_vidyo/cli.py new file mode 100644 index 0000000..b995f7e --- /dev/null +++ b/vc_vidyo/indico_vc_vidyo/cli.py @@ -0,0 +1,71 @@ +# This file is part of Indico. +# Copyright (C) 2002 - 2015 European Organization for Nuclear Research (CERN). +# +# Indico is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License as +# published by the Free Software Foundation; either version 3 of the +# License, or (at your option) any later version. +# +# Indico is distributed in the hope that it will be useful, but +# WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with Indico; if not, see . + +from __future__ import unicode_literals + +import sys + +from dateutil import rrule +from flask_script import Manager +from terminaltables import AsciiTable + +from indico.core.db import db, DBMgr +from indico.core.db.sqlalchemy.util.session import update_session_options +from indico.modules.scheduler import Client +from indico.modules.vc.models.vc_rooms import VCRoom, VCRoomStatus + +from indico_vc_vidyo.task import VidyoCleanupTask + +cli_manager = Manager(usage="Manages the Vidyo plugin") + + +@cli_manager.command +@cli_manager.option('--deleted', action='store_true') +@cli_manager.option('--created', action='store_true') +def rooms(deleted=False, created=False): + """Lists all Vidyo rooms""" + + room_query = VCRoom.find(type='vidyo') + table_data = [['ID', 'Name', 'Status', 'Vidyo ID', 'Extension']] + + if deleted: + room_query = room_query.filter(VCRoom.status == VCRoomStatus.deleted) + if created: + room_query = room_query.filter(VCRoom.status == VCRoomStatus.created) + + for room in room_query: + table_data.append([unicode(room.id), room.name, room.status.name, + unicode(room.data['vidyo_id']), unicode(room.vidyo_extension.extension)]) + + table = AsciiTable(table_data) + table.justify_columns[4] = 'right' + print table.table + + +@cli_manager.command +def create_task(interval): + """Creates a Vidyo cleanup task running every N days""" + update_session_options(db) + try: + interval = int(interval) + if interval < 1: + raise ValueError + except ValueError: + print 'Invalid interval, must be a number >=1' + sys.exit(1) + with DBMgr.getInstance().global_connection(commit=True): + Client().enqueue(VidyoCleanupTask(rrule.DAILY, interval=interval)) + print 'Task created' diff --git a/vc_vidyo/indico_vc_vidyo/plugin.py b/vc_vidyo/indico_vc_vidyo/plugin.py index e21a15d..65a037d 100644 --- a/vc_vidyo/indico_vc_vidyo/plugin.py +++ b/vc_vidyo/indico_vc_vidyo/plugin.py @@ -24,7 +24,7 @@ from wtforms.fields.simple import StringField from wtforms.validators import NumberRange, DataRequired from indico.core.config import Config -from indico.core.plugins import IndicoPlugin, url_for_plugin, IndicoPluginBlueprint +from indico.core.plugins import IndicoPlugin, url_for_plugin, IndicoPluginBlueprint, wrap_cli_manager from indico.modules.vc.exceptions import VCRoomError, VCRoomNotFoundError from indico.modules.vc import VCPluginSettingsFormBase, VCPluginMixin from indico.modules.vc.views import WPVCManageEvent, WPVCEventPage @@ -34,6 +34,7 @@ from indico.web.forms.fields import IndicoPasswordField from indico.web.forms.widgets import CKEditorWidget from indico_vc_vidyo.api import AdminClient, APIException, RoomNotFoundAPIException +from indico_vc_vidyo.cli import cli_manager from indico_vc_vidyo.forms import VCRoomForm, VCRoomAttachForm from indico_vc_vidyo.util import iter_user_identities, iter_extensions, update_room_from_obj from indico_vc_vidyo.models.vidyo_extensions import VidyoExtension @@ -275,12 +276,15 @@ class VidyoPlugin(VCPluginMixin, IndicoPlugin): client = AdminClient(self.settings) return client.get_room(vc_room.data['vidyo_id']) + def get_blueprints(self): + return IndicoPluginBlueprint('vc_vidyo', __name__) + def register_assets(self): self.register_css_bundle('vc_vidyo_css', 'css/vc_vidyo.scss') self.register_js_bundle('vc_vidyo_js', 'js/vc_vidyo.js') - def get_blueprints(self): - return IndicoPluginBlueprint('vc_vidyo', __name__) + def add_cli_command(self, manager): + manager.add_command('vidyo', wrap_cli_manager(cli_manager, self)) def get_vc_room_form_defaults(self, event): defaults = super(VidyoPlugin, self).get_vc_room_form_defaults(event) diff --git a/vc_vidyo/indico_vc_vidyo/task.py b/vc_vidyo/indico_vc_vidyo/task.py new file mode 100644 index 0000000..5d7e90c --- /dev/null +++ b/vc_vidyo/indico_vc_vidyo/task.py @@ -0,0 +1,82 @@ +# This file is part of Indico. +# Copyright (C) 2002 - 2015 European Organization for Nuclear Research (CERN). +# +# Indico is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License as +# published by the Free Software Foundation; either version 3 of the +# License, or (at your option) any later version. +# +# Indico is distributed in the hope that it will be useful, but +# WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with Indico; if not, see . + +from __future__ import unicode_literals + +from datetime import timedelta + +import sqlalchemy +from sqlalchemy.sql.expression import cast + +from indico.core.db import db +from indico.modules.vc.models.vc_rooms import VCRoomEventAssociation, VCRoom, VCRoomStatus +from indico.modules.fulltextindexes.models.events import IndexedEvent +from indico.modules.scheduler.tasks.periodic import PeriodicUniqueTask +from indico.util.date_time import now_utc +from indico.util.struct.iterables import committing_iterator +from indico_vc_vidyo.api import RoomNotFoundAPIException + + +def find_old_vidyo_rooms(max_room_event_age): + """Finds all Vidyo rooms that are: + - linked to no events + - linked only to events whose start date precedes today - max_room_event_age days + """ + recently_used = db.session.query(VCRoom.id).filter( + VCRoom.type == 'vidyo', + IndexedEvent.end_date > (now_utc() - timedelta(days=max_room_event_age)) + ).join(VCRoomEventAssociation).join( + IndexedEvent, IndexedEvent.id == cast(VCRoomEventAssociation.event_id, sqlalchemy.String) + ).group_by(VCRoom.id) + + # non-deleted rooms with no recent associations + return VCRoom.find(VCRoom.status != VCRoomStatus.deleted, ~VCRoom.id.in_(recently_used)).all() + + +class VidyoCleanupTask(PeriodicUniqueTask): + """Gets rid of 'old' Vidyo rooms (not used in recent events) + """ + DISABLE_ZODB_HOOK = True + + @property + def logger(self): + return self.getLogger() + + def run(self): + from indico_vc_vidyo.plugin import VidyoPlugin + + plugin = VidyoPlugin.instance # RuntimeError if not active + with plugin.plugin_context(): + + max_room_event_age = plugin.settings.get('num_days_old') + + self.logger.info('Deleting Vidyo rooms that are not used or linked to events all older than {} days'.format( + max_room_event_age)) + + candidate_rooms = find_old_vidyo_rooms(max_room_event_age) + + self.logger.info('{} rooms found'.format(len(candidate_rooms))) + + for vc_room in committing_iterator(candidate_rooms, n=20): + try: + plugin.delete_room(vc_room, None) + self.logger.info('Room {} deleted from Vidyo server'.format(vc_room)) + vc_room.status = VCRoomStatus.deleted + except RoomNotFoundAPIException: + self.logger.warning('Room {} had been already deleted from the Vidyo server'.format(vc_room)) + vc_room.status = VCRoomStatus.deleted + except: + self.logger.exception('Impossible to delete Vidyo room {}'.format(vc_room)) diff --git a/vc_vidyo/setup.py b/vc_vidyo/setup.py index a332e97..42de069 100644 --- a/vc_vidyo/setup.py +++ b/vc_vidyo/setup.py @@ -32,6 +32,7 @@ setup( platforms='any', install_requires=[ 'indico>=1.9.1', + 'terminaltables==1.0.2', 'suds-jurko' ], classifiers=[ diff --git a/vc_vidyo/tests/task_test.py b/vc_vidyo/tests/task_test.py new file mode 100644 index 0000000..2ee4df2 --- /dev/null +++ b/vc_vidyo/tests/task_test.py @@ -0,0 +1,91 @@ +# This file is part of Indico. +# Copyright (C) 2002 - 2015 European Organization for Nuclear Research (CERN). +# +# Indico is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License as +# published by the Free Software Foundation; either version 3 of the +# License, or (at your option) any later version. +# +# Indico is distributed in the hope that it will be useful, but +# WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with Indico; if not, see . + +from datetime import datetime + +import pytest + +from indico.modules.fulltextindexes.models.events import IndexedEvent +from indico.modules.vc.models.vc_rooms import VCRoom, VCRoomEventAssociation, VCRoomStatus, VCRoomLinkType +from indico_vc_vidyo.models.vidyo_extensions import VidyoExtension + + +class DummyEvent(object): + def __init__(self, id_, title, end_date): + self.id = id_ + self.title = title + self.end_date = end_date + + def getEndDate(self): + return self.end_date + + def getId(self): + return self.id + + +@pytest.fixture +def create_dummy_room(db, dummy_user): + """Returns a callable which lets you create dummy Vidyo room occurrences""" + def _create_room(name, extension, owner, data, **kwargs): + vc_room = VCRoom( + name=name, + data=data, + type="vidyo", + status=kwargs.pop('status', VCRoomStatus.created), + created_by_id=kwargs.pop('created_by_id', dummy_user.id), + **kwargs) + db.session.add(vc_room) + db.session.flush() + extension = VidyoExtension( + vc_room_id=vc_room.id, + extension=extension, + owned_by_id=owner.id + ) + vc_room.vidyo_extension = extension + return vc_room + + return _create_room + + +def test_room_cleanup(create_dummy_room, dummy_user, freeze_time, db): + """Test that 'old' Vidyo rooms are correctly detected""" + freeze_time(datetime(2015, 2, 1)) + events = {} + + for id_, args in enumerate((('Event one', datetime(2012, 1, 1)), + ('Event two', datetime(2013, 1, 1)), + ('Event three', datetime(2014, 1, 1)), + ('Event four', datetime(2015, 1, 1))), start=1): + event = DummyEvent(id_, *args) + events[id_] = event + idx = IndexedEvent(id=id_, title=args[0], end_date=args[1], start_date=args[1]) + db.session.add(idx) + + for id_, args in enumerate(((1234, 5678, (1, 2, 3, 4)), + (1235, 5679, (1, 2)), + (1235, 5679, (2,)), + (1236, 5670, (4,)), + (1237, 5671, ())), start=1): + room = create_dummy_room('test_room_{}'.format(id_), args[0], dummy_user, { + 'vidyo_id': args[1] + }) + for evt_id in args[2]: + VCRoomEventAssociation(vc_room=room, event=events[evt_id], link_type=VCRoomLinkType.event) + db.session.flush() + + from indico_vc_vidyo.task import find_old_vidyo_rooms + + assert [r.id for r in find_old_vidyo_rooms(180)] == [2, 3, 5]