From 3e946c78d03d0166401364b76de925063a0b320c Mon Sep 17 00:00:00 2001 From: jeremystretch Date: Mon, 20 Feb 2023 09:02:58 -0500 Subject: [PATCH 1/9] #11685: Clear cached search records for relevant IPAM objects --- .../migrations/0064_clear_search_cache.py | 28 +++++++++++++++++++ 1 file changed, 28 insertions(+) create mode 100644 netbox/ipam/migrations/0064_clear_search_cache.py diff --git a/netbox/ipam/migrations/0064_clear_search_cache.py b/netbox/ipam/migrations/0064_clear_search_cache.py new file mode 100644 index 000000000..92d2b2cd7 --- /dev/null +++ b/netbox/ipam/migrations/0064_clear_search_cache.py @@ -0,0 +1,28 @@ +from django.db import migrations + + +def clear_cache(apps, schema_editor): + """ + Clear existing CachedValues referencing IPAddressFields or IPNetworkFields. (#11658 + introduced new cache record types for these.) + """ + ContentType = apps.get_model('contenttypes', 'ContentType') + CachedValue = apps.get_model('extras', 'CachedValue') + + for model_name in ('Aggregate', 'IPAddress', 'IPRange', 'Prefix'): + content_type = ContentType.objects.get(app_label='ipam', model=model_name.lower()) + CachedValue.objects.filter(object_type=content_type).delete() + + +class Migration(migrations.Migration): + + dependencies = [ + ('ipam', '0063_standardize_description_comments'), + ] + + operations = [ + migrations.RunPython( + code=clear_cache, + reverse_code=migrations.RunPython.noop + ), + ] From 5a4d8a71075a4529be8d5ef22d76d69861240f45 Mon Sep 17 00:00:00 2001 From: jeremystretch Date: Mon, 20 Feb 2023 09:39:33 -0500 Subject: [PATCH 2/9] Closes #11787: Rebuild any missing search cache entires after upgrade --- docs/release-notes/version-3.4.md | 2 ++ netbox/extras/management/commands/reindex.py | 31 ++++++++++++++------ netbox/netbox/search/backends.py | 14 ++++++++- upgrade.sh | 5 ++++ 4 files changed, 42 insertions(+), 10 deletions(-) diff --git a/docs/release-notes/version-3.4.md b/docs/release-notes/version-3.4.md index 485b85719..a006eb7bf 100644 --- a/docs/release-notes/version-3.4.md +++ b/docs/release-notes/version-3.4.md @@ -7,6 +7,7 @@ * [#11110](https://github.com/netbox-community/netbox/issues/11110) - Add `start_address` and `end_address` filters for IP ranges * [#11592](https://github.com/netbox-community/netbox/issues/11592) - Introduce `FILE_UPLOAD_MAX_MEMORY_SIZE` configuration parameter * [#11685](https://github.com/netbox-community/netbox/issues/11685) - Match on containing prefixes and aggregates when querying for IP addresses using global search +* [#11787](https://github.com/netbox-community/netbox/issues/11787) - Upgrade script will automatically rebuild missing search cache ### Bug Fixes @@ -20,6 +21,7 @@ * [#11683](https://github.com/netbox-community/netbox/issues/11683) - Fix CSV header attribute detection when auto-detecting import format * [#11711](https://github.com/netbox-community/netbox/issues/11711) - Fix CSV import for multiple-object custom fields * [#11723](https://github.com/netbox-community/netbox/issues/11723) - Circuit terminations should link to their associated circuits (rather than site or provider network) +* [#11775](https://github.com/netbox-community/netbox/issues/11775) - Skip checking for old search cache records when creating a new object * [#11786](https://github.com/netbox-community/netbox/issues/11786) - List only applicable object types in form widget when filtering custom fields --- diff --git a/netbox/extras/management/commands/reindex.py b/netbox/extras/management/commands/reindex.py index b601a1ac1..9a29c54f5 100644 --- a/netbox/extras/management/commands/reindex.py +++ b/netbox/extras/management/commands/reindex.py @@ -15,6 +15,11 @@ class Command(BaseCommand): nargs='*', help='One or more apps or models to reindex', ) + parser.add_argument( + '--lazy', + action='store_true', + help="For each model, reindex objects only if no cache entries already exist" + ) def _get_indexers(self, *model_names): indexers = {} @@ -60,14 +65,15 @@ class Command(BaseCommand): raise CommandError("No indexers found!") self.stdout.write(f'Reindexing {len(indexers)} models.') - # Clear all cached values for the specified models - self.stdout.write('Clearing cached values... ', ending='') - self.stdout.flush() - content_types = [ - ContentType.objects.get_for_model(model) for model in indexers.keys() - ] - deleted_count = search_backend.clear(content_types) - self.stdout.write(f'{deleted_count} entries deleted.') + # Clear all cached values for the specified models (if not being lazy) + if not kwargs['lazy']: + self.stdout.write('Clearing cached values... ', ending='') + self.stdout.flush() + content_types = [ + ContentType.objects.get_for_model(model) for model in indexers.keys() + ] + deleted_count = search_backend.clear(content_types) + self.stdout.write(f'{deleted_count} entries deleted.') # Index models self.stdout.write('Indexing models') @@ -76,11 +82,18 @@ class Command(BaseCommand): model_name = model._meta.model_name self.stdout.write(f' {app_label}.{model_name}... ', ending='') self.stdout.flush() + + if kwargs['lazy']: + content_type = ContentType.objects.get_for_model(model) + if cached_count := search_backend.count(object_types=[content_type]): + self.stdout.write(f'Skipping (found {cached_count} existing).') + continue + i = search_backend.cache(model.objects.iterator(), remove_existing=False) if i: self.stdout.write(f'{i} entries cached.') else: - self.stdout.write(f'None found.') + self.stdout.write(f'No objects found.') msg = f'Completed.' if total_count := search_backend.size: diff --git a/netbox/netbox/search/backends.py b/netbox/netbox/search/backends.py index 14b5a987c..f428842f5 100644 --- a/netbox/netbox/search/backends.py +++ b/netbox/netbox/search/backends.py @@ -80,7 +80,13 @@ class SearchBackend: def clear(self, object_types=None): """ - Delete *all* cached data. + Delete *all* cached data (optionally filtered by object type). + """ + raise NotImplementedError + + def count(self, object_types=None): + """ + Return a count of all cache entries (optionally filtered by object type). """ raise NotImplementedError @@ -218,6 +224,12 @@ class CachedValueSearchBackend(SearchBackend): # Call _raw_delete() on the queryset to avoid first loading instances into memory return qs._raw_delete(using=qs.db) + def count(self, object_types=None): + qs = CachedValue.objects.all() + if object_types: + qs = qs.filter(object_type__in=object_types) + return qs.count() + @property def size(self): return CachedValue.objects.count() diff --git a/upgrade.sh b/upgrade.sh index 161d65e32..cac046a9f 100755 --- a/upgrade.sh +++ b/upgrade.sh @@ -103,6 +103,11 @@ COMMAND="python3 netbox/manage.py remove_stale_contenttypes --no-input" echo "Removing stale content types ($COMMAND)..." eval $COMMAND || exit 1 +# Rebuild the search cache (lazily) +COMMAND="python3 netbox/manage.py reindex --lazy" +echo "Rebuilding search cache ($COMMAND)..." +eval $COMMAND || exit 1 + # Delete any expired user sessions COMMAND="python3 netbox/manage.py clearsessions" echo "Removing expired user sessions ($COMMAND)..." From 972ba7bfdc16b42f68b3172ad605a3e113e4ea79 Mon Sep 17 00:00:00 2001 From: jeremystretch Date: Mon, 20 Feb 2023 10:27:30 -0500 Subject: [PATCH 3/9] #11685: Fix migration --- netbox/ipam/migrations/0064_clear_search_cache.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/netbox/ipam/migrations/0064_clear_search_cache.py b/netbox/ipam/migrations/0064_clear_search_cache.py index 92d2b2cd7..856fe99e1 100644 --- a/netbox/ipam/migrations/0064_clear_search_cache.py +++ b/netbox/ipam/migrations/0064_clear_search_cache.py @@ -10,8 +10,11 @@ def clear_cache(apps, schema_editor): CachedValue = apps.get_model('extras', 'CachedValue') for model_name in ('Aggregate', 'IPAddress', 'IPRange', 'Prefix'): - content_type = ContentType.objects.get(app_label='ipam', model=model_name.lower()) - CachedValue.objects.filter(object_type=content_type).delete() + try: + content_type = ContentType.objects.get(app_label='ipam', model=model_name.lower()) + CachedValue.objects.filter(object_type=content_type).delete() + except ContentType.DoesNotExist: + pass class Migration(migrations.Migration): From 3586cf79d4cebcff81ab46fc471b32930ce2d0f1 Mon Sep 17 00:00:00 2001 From: jeremystretch Date: Tue, 21 Feb 2023 08:42:39 -0500 Subject: [PATCH 4/9] Arrange parameters alphabetically --- netbox/netbox/configuration_example.py | 34 ++++++++++++-------------- 1 file changed, 15 insertions(+), 19 deletions(-) diff --git a/netbox/netbox/configuration_example.py b/netbox/netbox/configuration_example.py index 7158308af..92f6133a3 100644 --- a/netbox/netbox/configuration_example.py +++ b/netbox/netbox/configuration_example.py @@ -107,6 +107,9 @@ CORS_ORIGIN_REGEX_WHITELIST = [ # r'^(https?://)?(\w+\.)?example\.com$', ] +# The name to use for the CSRF token cookie. +CSRF_COOKIE_NAME = 'csrftoken' + # Set to True to enable server debugging. WARNING: Debugging introduces a substantial performance penalty and may reveal # sensitive information about your installation. Only enable debugging while performing testing. Never enable debugging # on a production system. @@ -127,6 +130,9 @@ EMAIL = { 'FROM_EMAIL': '', } +# Localization +ENABLE_LOCALIZATION = False + # Exempt certain models from the enforcement of view permissions. Models listed here will be viewable by all users and # by anonymous users. List models in the form `.`. Add '*' to this list to exempt all models. EXEMPT_VIEW_PERMISSIONS = [ @@ -168,16 +174,6 @@ LOGOUT_REDIRECT_URL = 'home' # the default value of this setting is derived from the installed location. # MEDIA_ROOT = '/opt/netbox/netbox/media' -# By default uploaded media is stored on the local filesystem. Using Django-storages is also supported. Provide the -# class path of the storage driver in STORAGE_BACKEND and any configuration options in STORAGE_CONFIG. For example: -# STORAGE_BACKEND = 'storages.backends.s3boto3.S3Boto3Storage' -# STORAGE_CONFIG = { -# 'AWS_ACCESS_KEY_ID': 'Key ID', -# 'AWS_SECRET_ACCESS_KEY': 'Secret', -# 'AWS_STORAGE_BUCKET_NAME': 'netbox', -# 'AWS_S3_REGION_NAME': 'eu-west-1', -# } - # Expose Prometheus monitoring metrics at the HTTP endpoint '/metrics' METRICS_ENABLED = False @@ -217,13 +213,6 @@ RQ_DEFAULT_TIMEOUT = 300 # this setting is derived from the installed location. # SCRIPTS_ROOT = '/opt/netbox/netbox/scripts' -# The maximum size (in bytes) that an upload will be before it gets streamed to the file system. -# Useful to be able to upload files bigger than 2.5Mbyte to custom scripts for processing. -# FILE_UPLOAD_MAX_MEMORY_SIZE = 2621440 - -# The name to use for the csrf token cookie. -CSRF_COOKIE_NAME = 'csrftoken' - # The name to use for the session cookie. SESSION_COOKIE_NAME = 'sessionid' @@ -232,8 +221,15 @@ SESSION_COOKIE_NAME = 'sessionid' # database access.) Note that the user as which NetBox runs must have read and write permissions to this path. SESSION_FILE_PATH = None -# Localization -ENABLE_LOCALIZATION = False +# By default, uploaded media is stored on the local filesystem. Using Django-storages is also supported. Provide the +# class path of the storage driver in STORAGE_BACKEND and any configuration options in STORAGE_CONFIG. For example: +# STORAGE_BACKEND = 'storages.backends.s3boto3.S3Boto3Storage' +# STORAGE_CONFIG = { +# 'AWS_ACCESS_KEY_ID': 'Key ID', +# 'AWS_SECRET_ACCESS_KEY': 'Secret', +# 'AWS_STORAGE_BUCKET_NAME': 'netbox', +# 'AWS_S3_REGION_NAME': 'eu-west-1', +# } # Time zone (default: UTC) TIME_ZONE = 'UTC' From c280ca35d69f1c968b8d454db04ae23e20ef338e Mon Sep 17 00:00:00 2001 From: jeremystretch Date: Tue, 21 Feb 2023 08:45:52 -0500 Subject: [PATCH 5/9] Release v3.4.5 --- .github/ISSUE_TEMPLATE/bug_report.yaml | 2 +- .github/ISSUE_TEMPLATE/feature_request.yaml | 2 +- docs/configuration/miscellaneous.md | 6 +++--- docs/release-notes/version-3.4.md | 3 +-- netbox/netbox/settings.py | 2 +- requirements.txt | 12 ++++++------ 6 files changed, 13 insertions(+), 14 deletions(-) diff --git a/.github/ISSUE_TEMPLATE/bug_report.yaml b/.github/ISSUE_TEMPLATE/bug_report.yaml index 9ed740fff..99c060e8a 100644 --- a/.github/ISSUE_TEMPLATE/bug_report.yaml +++ b/.github/ISSUE_TEMPLATE/bug_report.yaml @@ -14,7 +14,7 @@ body: attributes: label: NetBox version description: What version of NetBox are you currently running? - placeholder: v3.4.4 + placeholder: v3.4.5 validations: required: true - type: dropdown diff --git a/.github/ISSUE_TEMPLATE/feature_request.yaml b/.github/ISSUE_TEMPLATE/feature_request.yaml index 8e4ab54a5..1338be9c1 100644 --- a/.github/ISSUE_TEMPLATE/feature_request.yaml +++ b/.github/ISSUE_TEMPLATE/feature_request.yaml @@ -14,7 +14,7 @@ body: attributes: label: NetBox version description: What version of NetBox are you currently running? - placeholder: v3.4.4 + placeholder: v3.4.5 validations: required: true - type: dropdown diff --git a/docs/configuration/miscellaneous.md b/docs/configuration/miscellaneous.md index eac5d0a2f..8550564d8 100644 --- a/docs/configuration/miscellaneous.md +++ b/docs/configuration/miscellaneous.md @@ -69,11 +69,11 @@ By default, NetBox will permit users to create duplicate prefixes and IP address --- -## FILE_UPLOAD_MAX_MEMORY_SIZE +## `FILE_UPLOAD_MAX_MEMORY_SIZE` -Default: 2621440 (i.e. 2.5 MB). +Default: `2621440` (2.5 MB). -The maximum size (in bytes) that an upload will be before it gets streamed to the file system. Changing this setting can be useful for example to be able to upload files bigger than 2.5MB to custom scripts for processing. +The maximum amount (in bytes) of uploaded data that will be held in memory before being written to the filesystem. Changing this setting can be useful for example to be able to upload files bigger than 2.5MB to custom scripts for processing. --- diff --git a/docs/release-notes/version-3.4.md b/docs/release-notes/version-3.4.md index a006eb7bf..11df3d47a 100644 --- a/docs/release-notes/version-3.4.md +++ b/docs/release-notes/version-3.4.md @@ -1,6 +1,6 @@ # NetBox v3.4 -## v3.4.5 (FUTURE) +## v3.4.5 (2023-02-21) ### Enhancements @@ -13,7 +13,6 @@ * [#11032](https://github.com/netbox-community/netbox/issues/11032) - Fix false custom validation errors during component creation * [#11226](https://github.com/netbox-community/netbox/issues/11226) - Ensure scripts and reports within submodules are automatically reloaded -* [#11335](https://github.com/netbox-community/netbox/issues/11335) - Avoid exception when rendering change log after uninstalling a plugin * [#11459](https://github.com/netbox-community/netbox/issues/11459) - Enable evaluating null values in custom validation rules * [#11473](https://github.com/netbox-community/netbox/issues/11473) - GraphQL requests specifying an invalid filter should return an empty queryset * [#11582](https://github.com/netbox-community/netbox/issues/11582) - Ensure form validation errors are displayed when adding virtual chassis members diff --git a/netbox/netbox/settings.py b/netbox/netbox/settings.py index f6ce7ff33..2d56a025a 100644 --- a/netbox/netbox/settings.py +++ b/netbox/netbox/settings.py @@ -24,7 +24,7 @@ from netbox.constants import RQ_QUEUE_DEFAULT, RQ_QUEUE_HIGH, RQ_QUEUE_LOW # Environment setup # -VERSION = '3.4.5-dev' +VERSION = '3.4.5' # Hostname HOSTNAME = platform.node() diff --git a/requirements.txt b/requirements.txt index 3cb2529a8..8bbb80d1e 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,5 +1,5 @@ bleach==5.0.1 -Django==4.1.6 +Django==4.1.7 django-cors-headers==3.13.0 django-debug-toolbar==3.8.1 django-filter==22.1 @@ -9,23 +9,23 @@ django-pglocks==1.0.4 django-prometheus==2.2.0 django-redis==5.2.0 django-rich==1.4.0 -django-rq==2.6.0 -django-tables2==2.5.1 +django-rq==2.7.0 +django-tables2==2.5.2 django-taggit==3.1.0 django-timezone-field==5.0 djangorestframework==3.14.0 -drf-yasg[validation]==1.21.4 +drf-yasg[validation]==1.21.5 graphene-django==3.0.0 gunicorn==20.1.0 Jinja2==3.1.2 Markdown==3.3.7 -mkdocs-material==9.0.10 +mkdocs-material==9.0.13 mkdocstrings[python-legacy]==0.20.0 netaddr==0.8.0 Pillow==9.4.0 psycopg2-binary==2.9.5 PyYAML==6.0 -sentry-sdk==1.14.0 +sentry-sdk==1.15.0 social-auth-app-django==5.0.0 social-auth-core[openidconnect]==4.3.0 svgwrite==1.4.3 From 561f1eadfc2386bd266b941d019d830c37c53cf3 Mon Sep 17 00:00:00 2001 From: jeremystretch Date: Tue, 21 Feb 2023 09:03:19 -0500 Subject: [PATCH 6/9] PRVB --- docs/release-notes/version-3.4.md | 4 ++++ netbox/netbox/settings.py | 2 +- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/docs/release-notes/version-3.4.md b/docs/release-notes/version-3.4.md index 11df3d47a..cb8ba1e6e 100644 --- a/docs/release-notes/version-3.4.md +++ b/docs/release-notes/version-3.4.md @@ -1,5 +1,9 @@ # NetBox v3.4 +## v3.4.6 (FUTURE) + +--- + ## v3.4.5 (2023-02-21) ### Enhancements diff --git a/netbox/netbox/settings.py b/netbox/netbox/settings.py index 2d56a025a..967766763 100644 --- a/netbox/netbox/settings.py +++ b/netbox/netbox/settings.py @@ -24,7 +24,7 @@ from netbox.constants import RQ_QUEUE_DEFAULT, RQ_QUEUE_HIGH, RQ_QUEUE_LOW # Environment setup # -VERSION = '3.4.5' +VERSION = '3.4.6-dev' # Hostname HOSTNAME = platform.node() From ed77c0383093345112af94c27c9fa1f848d6dc37 Mon Sep 17 00:00:00 2001 From: jeremystretch Date: Mon, 27 Feb 2023 08:26:32 -0500 Subject: [PATCH 7/9] Fixes #11796: When importing devices, restrict rack by location only if the location field is specified --- docs/release-notes/version-3.4.md | 4 ++++ netbox/dcim/forms/bulk_import.py | 7 +++++-- 2 files changed, 9 insertions(+), 2 deletions(-) diff --git a/docs/release-notes/version-3.4.md b/docs/release-notes/version-3.4.md index cb8ba1e6e..e1d719dc6 100644 --- a/docs/release-notes/version-3.4.md +++ b/docs/release-notes/version-3.4.md @@ -2,6 +2,10 @@ ## v3.4.6 (FUTURE) +### Bug Fixes + +* [#11796](https://github.com/netbox-community/netbox/issues/11796) - When importing devices, restrict rack by location only if the location field is specified + --- ## v3.4.5 (2023-02-21) diff --git a/netbox/dcim/forms/bulk_import.py b/netbox/dcim/forms/bulk_import.py index 3f016899e..da658d732 100644 --- a/netbox/dcim/forms/bulk_import.py +++ b/netbox/dcim/forms/bulk_import.py @@ -447,11 +447,14 @@ class DeviceImportForm(BaseDeviceImportForm): self.fields['location'].queryset = self.fields['location'].queryset.filter(**params) self.fields['parent'].queryset = self.fields['parent'].queryset.filter(**params) - # Limit rack queryset by assigned site and group + # Limit rack queryset by assigned site and location params = { f"site__{self.fields['site'].to_field_name}": data.get('site'), - f"location__{self.fields['location'].to_field_name}": data.get('location'), } + if 'location' in data: + params.update({ + f"location__{self.fields['location'].to_field_name}": data.get('location'), + }) self.fields['rack'].queryset = self.fields['rack'].queryset.filter(**params) # Limit device bay queryset by parent device From cfa6b28ceb8667bc60f2924969f56be1d828f136 Mon Sep 17 00:00:00 2001 From: jeremystretch Date: Mon, 27 Feb 2023 09:22:48 -0500 Subject: [PATCH 8/9] Closes #11807: Restore default page size when navigating between views --- docs/release-notes/version-3.4.md | 4 ++++ netbox/netbox/preferences.py | 2 +- netbox/utilities/paginator.py | 2 -- 3 files changed, 5 insertions(+), 3 deletions(-) diff --git a/docs/release-notes/version-3.4.md b/docs/release-notes/version-3.4.md index e1d719dc6..484f5bab7 100644 --- a/docs/release-notes/version-3.4.md +++ b/docs/release-notes/version-3.4.md @@ -2,6 +2,10 @@ ## v3.4.6 (FUTURE) +### Enhancements + +* [#11807](https://github.com/netbox-community/netbox/issues/11807) - Restore default page size when navigating between views + ### Bug Fixes * [#11796](https://github.com/netbox-community/netbox/issues/11796) - When importing devices, restrict rack by location only if the location field is specified diff --git a/netbox/netbox/preferences.py b/netbox/netbox/preferences.py index c88b56072..5ef216259 100644 --- a/netbox/netbox/preferences.py +++ b/netbox/netbox/preferences.py @@ -24,7 +24,7 @@ PREFERENCES = { 'pagination.per_page': UserPreference( label=_('Page length'), choices=get_page_lengths(), - description=_('The number of objects to display per page'), + description=_('The default number of objects to display per page'), coerce=lambda x: int(x) ), 'pagination.placement': UserPreference( diff --git a/netbox/utilities/paginator.py b/netbox/utilities/paginator.py index 1f07aa42a..db6326a9c 100644 --- a/netbox/utilities/paginator.py +++ b/netbox/utilities/paginator.py @@ -76,8 +76,6 @@ def get_paginate_count(request): if 'per_page' in request.GET: try: per_page = int(request.GET.get('per_page')) - if request.user.is_authenticated: - request.user.config.set('pagination.per_page', per_page, commit=True) return _max_allowed(per_page) except ValueError: pass From 0744ff2fa014ee4d23f89d588f35506c93a1ab60 Mon Sep 17 00:00:00 2001 From: Rafael Ganascim <67799240+rganascim@users.noreply.github.com> Date: Mon, 27 Feb 2023 13:42:30 -0300 Subject: [PATCH 9/9] Fixes #11758 - replace unsafe chars in menu label (#11831) * Fixes #11758 - replace unsafe chars in menu label * Fixes #11758 - replace unsafe chars in menu label --- netbox/extras/plugins/navigation.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/netbox/extras/plugins/navigation.py b/netbox/extras/plugins/navigation.py index e667965b8..288a78512 100644 --- a/netbox/extras/plugins/navigation.py +++ b/netbox/extras/plugins/navigation.py @@ -1,5 +1,6 @@ from netbox.navigation import MenuGroup from utilities.choices import ButtonColorChoices +from django.utils.text import slugify __all__ = ( 'PluginMenu', @@ -21,7 +22,7 @@ class PluginMenu: @property def name(self): - return self.label.replace(' ', '_') + return slugify(self.label) class PluginMenuItem: