diff --git a/.github/ISSUE_TEMPLATE/bug_report.yaml b/.github/ISSUE_TEMPLATE/bug_report.yaml index 9ed740fff..99c060e8a 100644 --- a/.github/ISSUE_TEMPLATE/bug_report.yaml +++ b/.github/ISSUE_TEMPLATE/bug_report.yaml @@ -14,7 +14,7 @@ body: attributes: label: NetBox version description: What version of NetBox are you currently running? - placeholder: v3.4.4 + placeholder: v3.4.5 validations: required: true - type: dropdown diff --git a/.github/ISSUE_TEMPLATE/feature_request.yaml b/.github/ISSUE_TEMPLATE/feature_request.yaml index 8e4ab54a5..1338be9c1 100644 --- a/.github/ISSUE_TEMPLATE/feature_request.yaml +++ b/.github/ISSUE_TEMPLATE/feature_request.yaml @@ -14,7 +14,7 @@ body: attributes: label: NetBox version description: What version of NetBox are you currently running? - placeholder: v3.4.4 + placeholder: v3.4.5 validations: required: true - type: dropdown diff --git a/docs/configuration/miscellaneous.md b/docs/configuration/miscellaneous.md index eac5d0a2f..8550564d8 100644 --- a/docs/configuration/miscellaneous.md +++ b/docs/configuration/miscellaneous.md @@ -69,11 +69,11 @@ By default, NetBox will permit users to create duplicate prefixes and IP address --- -## FILE_UPLOAD_MAX_MEMORY_SIZE +## `FILE_UPLOAD_MAX_MEMORY_SIZE` -Default: 2621440 (i.e. 2.5 MB). +Default: `2621440` (2.5 MB). -The maximum size (in bytes) that an upload will be before it gets streamed to the file system. Changing this setting can be useful for example to be able to upload files bigger than 2.5MB to custom scripts for processing. +The maximum amount (in bytes) of uploaded data that will be held in memory before being written to the filesystem. Changing this setting can be useful for example to be able to upload files bigger than 2.5MB to custom scripts for processing. --- diff --git a/docs/release-notes/version-3.4.md b/docs/release-notes/version-3.4.md index 485b85719..484f5bab7 100644 --- a/docs/release-notes/version-3.4.md +++ b/docs/release-notes/version-3.4.md @@ -1,18 +1,30 @@ # NetBox v3.4 -## v3.4.5 (FUTURE) +## v3.4.6 (FUTURE) + +### Enhancements + +* [#11807](https://github.com/netbox-community/netbox/issues/11807) - Restore default page size when navigating between views + +### Bug Fixes + +* [#11796](https://github.com/netbox-community/netbox/issues/11796) - When importing devices, restrict rack by location only if the location field is specified + +--- + +## v3.4.5 (2023-02-21) ### Enhancements * [#11110](https://github.com/netbox-community/netbox/issues/11110) - Add `start_address` and `end_address` filters for IP ranges * [#11592](https://github.com/netbox-community/netbox/issues/11592) - Introduce `FILE_UPLOAD_MAX_MEMORY_SIZE` configuration parameter * [#11685](https://github.com/netbox-community/netbox/issues/11685) - Match on containing prefixes and aggregates when querying for IP addresses using global search +* [#11787](https://github.com/netbox-community/netbox/issues/11787) - Upgrade script will automatically rebuild missing search cache ### Bug Fixes * [#11032](https://github.com/netbox-community/netbox/issues/11032) - Fix false custom validation errors during component creation * [#11226](https://github.com/netbox-community/netbox/issues/11226) - Ensure scripts and reports within submodules are automatically reloaded -* [#11335](https://github.com/netbox-community/netbox/issues/11335) - Avoid exception when rendering change log after uninstalling a plugin * [#11459](https://github.com/netbox-community/netbox/issues/11459) - Enable evaluating null values in custom validation rules * [#11473](https://github.com/netbox-community/netbox/issues/11473) - GraphQL requests specifying an invalid filter should return an empty queryset * [#11582](https://github.com/netbox-community/netbox/issues/11582) - Ensure form validation errors are displayed when adding virtual chassis members @@ -20,6 +32,7 @@ * [#11683](https://github.com/netbox-community/netbox/issues/11683) - Fix CSV header attribute detection when auto-detecting import format * [#11711](https://github.com/netbox-community/netbox/issues/11711) - Fix CSV import for multiple-object custom fields * [#11723](https://github.com/netbox-community/netbox/issues/11723) - Circuit terminations should link to their associated circuits (rather than site or provider network) +* [#11775](https://github.com/netbox-community/netbox/issues/11775) - Skip checking for old search cache records when creating a new object * [#11786](https://github.com/netbox-community/netbox/issues/11786) - List only applicable object types in form widget when filtering custom fields --- diff --git a/netbox/dcim/forms/bulk_import.py b/netbox/dcim/forms/bulk_import.py index 3f016899e..da658d732 100644 --- a/netbox/dcim/forms/bulk_import.py +++ b/netbox/dcim/forms/bulk_import.py @@ -447,11 +447,14 @@ class DeviceImportForm(BaseDeviceImportForm): self.fields['location'].queryset = self.fields['location'].queryset.filter(**params) self.fields['parent'].queryset = self.fields['parent'].queryset.filter(**params) - # Limit rack queryset by assigned site and group + # Limit rack queryset by assigned site and location params = { f"site__{self.fields['site'].to_field_name}": data.get('site'), - f"location__{self.fields['location'].to_field_name}": data.get('location'), } + if 'location' in data: + params.update({ + f"location__{self.fields['location'].to_field_name}": data.get('location'), + }) self.fields['rack'].queryset = self.fields['rack'].queryset.filter(**params) # Limit device bay queryset by parent device diff --git a/netbox/extras/management/commands/reindex.py b/netbox/extras/management/commands/reindex.py index b601a1ac1..9a29c54f5 100644 --- a/netbox/extras/management/commands/reindex.py +++ b/netbox/extras/management/commands/reindex.py @@ -15,6 +15,11 @@ class Command(BaseCommand): nargs='*', help='One or more apps or models to reindex', ) + parser.add_argument( + '--lazy', + action='store_true', + help="For each model, reindex objects only if no cache entries already exist" + ) def _get_indexers(self, *model_names): indexers = {} @@ -60,14 +65,15 @@ class Command(BaseCommand): raise CommandError("No indexers found!") self.stdout.write(f'Reindexing {len(indexers)} models.') - # Clear all cached values for the specified models - self.stdout.write('Clearing cached values... ', ending='') - self.stdout.flush() - content_types = [ - ContentType.objects.get_for_model(model) for model in indexers.keys() - ] - deleted_count = search_backend.clear(content_types) - self.stdout.write(f'{deleted_count} entries deleted.') + # Clear all cached values for the specified models (if not being lazy) + if not kwargs['lazy']: + self.stdout.write('Clearing cached values... ', ending='') + self.stdout.flush() + content_types = [ + ContentType.objects.get_for_model(model) for model in indexers.keys() + ] + deleted_count = search_backend.clear(content_types) + self.stdout.write(f'{deleted_count} entries deleted.') # Index models self.stdout.write('Indexing models') @@ -76,11 +82,18 @@ class Command(BaseCommand): model_name = model._meta.model_name self.stdout.write(f' {app_label}.{model_name}... ', ending='') self.stdout.flush() + + if kwargs['lazy']: + content_type = ContentType.objects.get_for_model(model) + if cached_count := search_backend.count(object_types=[content_type]): + self.stdout.write(f'Skipping (found {cached_count} existing).') + continue + i = search_backend.cache(model.objects.iterator(), remove_existing=False) if i: self.stdout.write(f'{i} entries cached.') else: - self.stdout.write(f'None found.') + self.stdout.write(f'No objects found.') msg = f'Completed.' if total_count := search_backend.size: diff --git a/netbox/extras/plugins/navigation.py b/netbox/extras/plugins/navigation.py index e667965b8..288a78512 100644 --- a/netbox/extras/plugins/navigation.py +++ b/netbox/extras/plugins/navigation.py @@ -1,5 +1,6 @@ from netbox.navigation import MenuGroup from utilities.choices import ButtonColorChoices +from django.utils.text import slugify __all__ = ( 'PluginMenu', @@ -21,7 +22,7 @@ class PluginMenu: @property def name(self): - return self.label.replace(' ', '_') + return slugify(self.label) class PluginMenuItem: diff --git a/netbox/ipam/migrations/0064_clear_search_cache.py b/netbox/ipam/migrations/0064_clear_search_cache.py new file mode 100644 index 000000000..856fe99e1 --- /dev/null +++ b/netbox/ipam/migrations/0064_clear_search_cache.py @@ -0,0 +1,31 @@ +from django.db import migrations + + +def clear_cache(apps, schema_editor): + """ + Clear existing CachedValues referencing IPAddressFields or IPNetworkFields. (#11658 + introduced new cache record types for these.) + """ + ContentType = apps.get_model('contenttypes', 'ContentType') + CachedValue = apps.get_model('extras', 'CachedValue') + + for model_name in ('Aggregate', 'IPAddress', 'IPRange', 'Prefix'): + try: + content_type = ContentType.objects.get(app_label='ipam', model=model_name.lower()) + CachedValue.objects.filter(object_type=content_type).delete() + except ContentType.DoesNotExist: + pass + + +class Migration(migrations.Migration): + + dependencies = [ + ('ipam', '0063_standardize_description_comments'), + ] + + operations = [ + migrations.RunPython( + code=clear_cache, + reverse_code=migrations.RunPython.noop + ), + ] diff --git a/netbox/netbox/configuration_example.py b/netbox/netbox/configuration_example.py index 7158308af..92f6133a3 100644 --- a/netbox/netbox/configuration_example.py +++ b/netbox/netbox/configuration_example.py @@ -107,6 +107,9 @@ CORS_ORIGIN_REGEX_WHITELIST = [ # r'^(https?://)?(\w+\.)?example\.com$', ] +# The name to use for the CSRF token cookie. +CSRF_COOKIE_NAME = 'csrftoken' + # Set to True to enable server debugging. WARNING: Debugging introduces a substantial performance penalty and may reveal # sensitive information about your installation. Only enable debugging while performing testing. Never enable debugging # on a production system. @@ -127,6 +130,9 @@ EMAIL = { 'FROM_EMAIL': '', } +# Localization +ENABLE_LOCALIZATION = False + # Exempt certain models from the enforcement of view permissions. Models listed here will be viewable by all users and # by anonymous users. List models in the form `.`. Add '*' to this list to exempt all models. EXEMPT_VIEW_PERMISSIONS = [ @@ -168,16 +174,6 @@ LOGOUT_REDIRECT_URL = 'home' # the default value of this setting is derived from the installed location. # MEDIA_ROOT = '/opt/netbox/netbox/media' -# By default uploaded media is stored on the local filesystem. Using Django-storages is also supported. Provide the -# class path of the storage driver in STORAGE_BACKEND and any configuration options in STORAGE_CONFIG. For example: -# STORAGE_BACKEND = 'storages.backends.s3boto3.S3Boto3Storage' -# STORAGE_CONFIG = { -# 'AWS_ACCESS_KEY_ID': 'Key ID', -# 'AWS_SECRET_ACCESS_KEY': 'Secret', -# 'AWS_STORAGE_BUCKET_NAME': 'netbox', -# 'AWS_S3_REGION_NAME': 'eu-west-1', -# } - # Expose Prometheus monitoring metrics at the HTTP endpoint '/metrics' METRICS_ENABLED = False @@ -217,13 +213,6 @@ RQ_DEFAULT_TIMEOUT = 300 # this setting is derived from the installed location. # SCRIPTS_ROOT = '/opt/netbox/netbox/scripts' -# The maximum size (in bytes) that an upload will be before it gets streamed to the file system. -# Useful to be able to upload files bigger than 2.5Mbyte to custom scripts for processing. -# FILE_UPLOAD_MAX_MEMORY_SIZE = 2621440 - -# The name to use for the csrf token cookie. -CSRF_COOKIE_NAME = 'csrftoken' - # The name to use for the session cookie. SESSION_COOKIE_NAME = 'sessionid' @@ -232,8 +221,15 @@ SESSION_COOKIE_NAME = 'sessionid' # database access.) Note that the user as which NetBox runs must have read and write permissions to this path. SESSION_FILE_PATH = None -# Localization -ENABLE_LOCALIZATION = False +# By default, uploaded media is stored on the local filesystem. Using Django-storages is also supported. Provide the +# class path of the storage driver in STORAGE_BACKEND and any configuration options in STORAGE_CONFIG. For example: +# STORAGE_BACKEND = 'storages.backends.s3boto3.S3Boto3Storage' +# STORAGE_CONFIG = { +# 'AWS_ACCESS_KEY_ID': 'Key ID', +# 'AWS_SECRET_ACCESS_KEY': 'Secret', +# 'AWS_STORAGE_BUCKET_NAME': 'netbox', +# 'AWS_S3_REGION_NAME': 'eu-west-1', +# } # Time zone (default: UTC) TIME_ZONE = 'UTC' diff --git a/netbox/netbox/preferences.py b/netbox/netbox/preferences.py index c88b56072..5ef216259 100644 --- a/netbox/netbox/preferences.py +++ b/netbox/netbox/preferences.py @@ -24,7 +24,7 @@ PREFERENCES = { 'pagination.per_page': UserPreference( label=_('Page length'), choices=get_page_lengths(), - description=_('The number of objects to display per page'), + description=_('The default number of objects to display per page'), coerce=lambda x: int(x) ), 'pagination.placement': UserPreference( diff --git a/netbox/netbox/search/backends.py b/netbox/netbox/search/backends.py index 14b5a987c..f428842f5 100644 --- a/netbox/netbox/search/backends.py +++ b/netbox/netbox/search/backends.py @@ -80,7 +80,13 @@ class SearchBackend: def clear(self, object_types=None): """ - Delete *all* cached data. + Delete *all* cached data (optionally filtered by object type). + """ + raise NotImplementedError + + def count(self, object_types=None): + """ + Return a count of all cache entries (optionally filtered by object type). """ raise NotImplementedError @@ -218,6 +224,12 @@ class CachedValueSearchBackend(SearchBackend): # Call _raw_delete() on the queryset to avoid first loading instances into memory return qs._raw_delete(using=qs.db) + def count(self, object_types=None): + qs = CachedValue.objects.all() + if object_types: + qs = qs.filter(object_type__in=object_types) + return qs.count() + @property def size(self): return CachedValue.objects.count() diff --git a/netbox/netbox/settings.py b/netbox/netbox/settings.py index f6ce7ff33..967766763 100644 --- a/netbox/netbox/settings.py +++ b/netbox/netbox/settings.py @@ -24,7 +24,7 @@ from netbox.constants import RQ_QUEUE_DEFAULT, RQ_QUEUE_HIGH, RQ_QUEUE_LOW # Environment setup # -VERSION = '3.4.5-dev' +VERSION = '3.4.6-dev' # Hostname HOSTNAME = platform.node() diff --git a/netbox/utilities/paginator.py b/netbox/utilities/paginator.py index 1f07aa42a..db6326a9c 100644 --- a/netbox/utilities/paginator.py +++ b/netbox/utilities/paginator.py @@ -76,8 +76,6 @@ def get_paginate_count(request): if 'per_page' in request.GET: try: per_page = int(request.GET.get('per_page')) - if request.user.is_authenticated: - request.user.config.set('pagination.per_page', per_page, commit=True) return _max_allowed(per_page) except ValueError: pass diff --git a/requirements.txt b/requirements.txt index 3cb2529a8..8bbb80d1e 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,5 +1,5 @@ bleach==5.0.1 -Django==4.1.6 +Django==4.1.7 django-cors-headers==3.13.0 django-debug-toolbar==3.8.1 django-filter==22.1 @@ -9,23 +9,23 @@ django-pglocks==1.0.4 django-prometheus==2.2.0 django-redis==5.2.0 django-rich==1.4.0 -django-rq==2.6.0 -django-tables2==2.5.1 +django-rq==2.7.0 +django-tables2==2.5.2 django-taggit==3.1.0 django-timezone-field==5.0 djangorestframework==3.14.0 -drf-yasg[validation]==1.21.4 +drf-yasg[validation]==1.21.5 graphene-django==3.0.0 gunicorn==20.1.0 Jinja2==3.1.2 Markdown==3.3.7 -mkdocs-material==9.0.10 +mkdocs-material==9.0.13 mkdocstrings[python-legacy]==0.20.0 netaddr==0.8.0 Pillow==9.4.0 psycopg2-binary==2.9.5 PyYAML==6.0 -sentry-sdk==1.14.0 +sentry-sdk==1.15.0 social-auth-app-django==5.0.0 social-auth-core[openidconnect]==4.3.0 svgwrite==1.4.3 diff --git a/upgrade.sh b/upgrade.sh index 161d65e32..cac046a9f 100755 --- a/upgrade.sh +++ b/upgrade.sh @@ -103,6 +103,11 @@ COMMAND="python3 netbox/manage.py remove_stale_contenttypes --no-input" echo "Removing stale content types ($COMMAND)..." eval $COMMAND || exit 1 +# Rebuild the search cache (lazily) +COMMAND="python3 netbox/manage.py reindex --lazy" +echo "Rebuilding search cache ($COMMAND)..." +eval $COMMAND || exit 1 + # Delete any expired user sessions COMMAND="python3 netbox/manage.py clearsessions" echo "Removing expired user sessions ($COMMAND)..."