Details
-
Type:
Bug
-
Status: Engineering Closed (View Workflow)
-
Priority:
High
-
Resolution: Need additional information
-
Affects Version/s: None
-
Fix Version/s: None
-
Component/s: Boot Environments
-
Labels:None
Description
I'm referencing this thread in the forums: https://www.ixsystems.com/community/threads/freenas-will-not-boot-stuck-on-middlewared-loaded-plugin-middleware-d-plugins-crypto.81692/#post-565641
I recently upgraded to 11.3RC2 - I could have sworn I had rebooted a few times with the upgrade from 11.2U7 to 11.3RC2 but today upon rebooting the system I reached a point where the system would boot then apparently get stuck displaying:
middleward: loaded plugin middlewared.plugins.crypto -- and then go no further
Per advice in the forums, I rebooted with 11.3RC2 boot environment and hit Cntl-C to proceed
I'm posting here my /var/log/middleware.log when booting into the unstable system:
# cat /var/log/middlewared.log
[2020/01/24 23:02:25] (DEBUG) raven.base.Client.set_dsn():272 - Configuring Raven for host: https://sentry.ixsystems.com
[2020/01/24 23:02:26] (ERROR) asyncio.default_exception_handler():1615 - Task exception was never retrieved
future: <Task finished coro=<Middleware.__initialize() done, defined at /usr/local/lib/python3.7/site-packages/middlewared/main.py:1298> exception=ModuleNotFoundError("No module named 'freenasUI.settings'")>
Traceback (most recent call last):
File "/usr/local/lib/python3.7/site-packages/middlewared/main.py", line 1305, in __initialize
setup_funcs = await self.__plugins_load()
File "/usr/local/lib/python3.7/site-packages/middlewared/main.py", line 797, in __plugins_load
on_modules_loaded=on_modules_loaded,
File "/usr/local/lib/python3.7/site-packages/middlewared/utils/__init__.py", line 504, in _load_plugins
for mod in load_modules(plugins_dir):
File "/usr/local/lib/python3.7/site-packages/middlewared/utils/__init__.py", line 463, in load_modules
yield imp.load_module(name, fp, pathname, description)
File "/usr/local/lib/python3.7/imp.py", line 234, in load_module
return load_source(name, filename, file)
File "/usr/local/lib/python3.7/imp.py", line 171, in load_source
module = _load(spec)
File "<frozen importlib._bootstrap>", line 696, in _load
File "<frozen importlib._bootstrap>", line 677, in _load_unlocked
File "<frozen importlib._bootstrap_external>", line 728, in exec_module
File "<frozen importlib._bootstrap>", line 219, in _call_with_frames_removed
File "/usr/local/lib/python3.7/site-packages/middlewared/plugins/datastore.py", line 15, in <module>
django.setup()
File "/usr/local/lib/python3.7/site-packages/django/__init__.py", line 22, in setup
configure_logging(settings.LOGGING_CONFIG, settings.LOGGING)
File "/usr/local/lib/python3.7/site-packages/django/conf/__init__.py", line 56, in __getattr__
self._setup(name)
File "/usr/local/lib/python3.7/site-packages/django/conf/__init__.py", line 41, in _setup
self._wrapped = Settings(settings_module)
File "/usr/local/lib/python3.7/site-packages/django/conf/__init__.py", line 110, in __init__
mod = importlib.import_module(self.SETTINGS_MODULE)
File "/usr/local/lib/python3.7/importlib/__init__.py", line 127, in import_module
return _bootstrap._gcd_import(name[level:], package, level)
File "<frozen importlib._bootstrap>", line 1006, in _gcd_import
File "<frozen importlib._bootstrap>", line 983, in _find_and_load
File "<frozen importlib._bootstrap>", line 965, in _find_and_load_unlocked
ModuleNotFoundError: No module named 'freenasUI.settings'I was able to roll back to 11.2U7 and everything seemed OK except for none of my jails iocage jails showing in the GUI (please note that warden jails still functioned properly).
iocage list gave following feedback:
# iocage list
Traceback (most recent call last):
File "/usr/local/bin/iocage", line 10, in <module>
sys.exit(cli())
File "/usr/local/lib/python3.6/site-packages/click/core.py", line 722, in __call__
return self.main(*args, **kwargs)
File "/usr/local/lib/python3.6/site-packages/click/core.py", line 697, in main
rv = self.invoke(ctx)
File "/usr/local/lib/python3.6/site-packages/click/core.py", line 1066, in invoke
return _process_result(sub_ctx.command.invoke(sub_ctx))
File "/usr/local/lib/python3.6/site-packages/click/core.py", line 895, in invoke
return ctx.invoke(self.callback, **ctx.params)
File "/usr/local/lib/python3.6/site-packages/click/core.py", line 535, in invoke
return callback(*args, **kwargs)
File "/usr/local/lib/python3.6/site-packages/iocage_cli/list.py", line 81, in cli
dataset_type, header, _long, _sort, plugin=plugins, quick=quick)
File "/usr/local/lib/python3.6/site-packages/iocage_lib/iocage.py", line 1169, in list
quick=quick
File "/usr/local/lib/python3.6/site-packages/iocage_lib/ioc_list.py", line 64, in list_datasets
iocage_lib.ioc_json.IOCJson().json_check_default_config()
File "/usr/local/lib/python3.6/site-packages/iocage_lib/ioc_json.py", line 1924, in json_check_default_config
self.json_write(default_props, default_json_location)
File "/usr/local/lib/python3.6/site-packages/iocage_lib/ioc_json.py", line 431, in json_write
if template else None
File "libzfs.pyx", line 645, in libzfs.ZFS.get_dataset_by_path
libzfs.ZFSException: Dataset with path not found
My /mnt/tank/iocage/defaults.json file on 11.2U7
{
"CONFIG_VERSION": "26",
"allow_chflags": 0,
"allow_mlock": 0,
"allow_mount": 0,
"allow_mount_devfs": 0,
"allow_mount_fusefs": 0,
"allow_mount_nullfs": 0,
"allow_mount_procfs": 0,
"allow_mount_tmpfs": 0,
"allow_mount_zfs": 0,
"allow_quotas": 0,
"allow_raw_sockets": 0,
"allow_set_hostname": 1,
"allow_socket_af": 0,
"allow_sysvipc": 0,
"allow_tun": 0,
"allow_vmm": 0,
"assign_localhost": 0,
"available": "readonly",
"basejail": 0,
"boot": 0,
"bpf": 0,
"children_max": "0",
"comment": "none",
"compression": "lz4",
"compressratio": "readonly",
"coredumpsize": "off",
"count": "1",
"cpuset": "off",
"cputime": "off",
"datasize": "off",
"dedup": "off",
"defaultrouter": "auto",
"defaultrouter6": "auto",
"depends": "none",
"devfs_ruleset": "4",
"dhcp": 0,
"enforce_statfs": "2",
"exec_clean": 1,
"exec_created": "/usr/bin/true",
"exec_fib": "0",
"exec_jail_user": "root",
"exec_poststart": "/usr/bin/true",
"exec_poststop": "/usr/bin/true",
"exec_prestart": "/usr/bin/true",
"exec_prestop": "/usr/bin/true",
"exec_start": "/bin/sh /etc/rc",
"exec_stop": "/bin/sh /etc/rc.shutdown",
"exec_system_jail_user": "0",
"exec_system_user": "root",
"exec_timeout": "60",
"host_domainname": "none",
"host_time": 1,
"hostid": "00000000-0000-0000-0000-0cc47a84a594",
"hostid_strict_check": 0,
"interfaces": "vnet0:bridge0",
"ip4": "new",
"ip4_addr": "none",
"ip4_saddrsel": 1,
"ip6": "new",
"ip6_addr": "none",
"ip6_saddrsel": 1,
"ip_hostname": 0,
"jail_zfs": 0,
"jail_zfs_mountpoint": "none",
"last_started": "none",
"localhost_ip": "none",
"login_flags": "-f root",
"mac_prefix": "02ff60",
"maxproc": "off",
"memorylocked": "off",
"memoryuse": "off",
"mount_devfs": 1,
"mount_fdescfs": 1,
"mount_linprocfs": 0,
"mount_procfs": 0,
"mountpoint": "readonly",
"msgqqueued": "off",
"msgqsize": "off",
"nat": 0,
"nat_backend": "ipfw",
"nat_forwards": "none",
"nat_interface": "none",
"nat_prefix": "172.16",
"nmsgq": "off",
"notes": "none",
"nsem": "off",
"nsemop": "off",
"nshm": "off",
"nthr": "off",
"openfiles": "off",
"origin": "readonly",
"owner": "root",
"pcpu": "off",
"plugin_name": "none",
"plugin_repository": "none",
"priority": "99",
"pseudoterminals": "off",
"quota": "none",
"readbps": "off",
"readiops": "off",
"reservation": "none",
"resolver": "/etc/resolv.conf",
"rlimits": "off",
"rtsold": 0,
"securelevel": "2",
"shmsize": "off",
"stacksize": "off",
"stop_timeout": "30",
"swapuse": "off",
"sync_state": "none",
"sync_target": "none",
"sync_tgt_zpool": "none",
"sysvmsg": "new",
"sysvsem": "new",
"sysvshm": "new",
"template": 0,
"type": "jail",
"used": "readonly",
"vmemoryuse": "off",
"vnet": 0,
"vnet0_mac": "none",
"vnet1_mac": "none",
"vnet2_mac": "none",
"vnet3_mac": "none",
"vnet_default_interface": "auto",
"vnet_interfaces": "none",
"wallclock": "off",
"writebps": "off",
"writeiops": "off"
}
I've attached debug log from 11.2U7
