11.3RC2 - Fails to boot - middleware.plugins.crypto

I'm referencing this thread in the forums: https://www.ixsystems.com/community/threads/freenas-will-not-boot-stuck-on-middlewared-loaded-plugin-middleware-d-plugins-crypto.81692/#post-565641

I recently upgraded to 11.3RC2 - I could have sworn I had rebooted a few times with the upgrade from 11.2U7 to 11.3RC2 but today upon rebooting the system I reached a point where the system would boot then apparently get stuck displaying:

middleward: loaded plugin middlewared.plugins.crypto – and then go no further

Per advice in the forums, I rebooted with 11.3RC2 boot environment and hit Cntl-C to proceed

I'm posting here my /var/log/middleware.log when booting into the unstable system:

1. cat /var/log/middlewared.log
   [2020/01/24 23:02:25] (DEBUG) raven.base.Client.set_dsn():272 - Configuring Raven for host: https://sentry.ixsystems.com
   [2020/01/24 23:02:26] (ERROR) asyncio.default_exception_handler():1615 - Task exception was never retrieved
   future: <Task finished coro=<Middleware.__initialize() done, defined at /usr/local/lib/python3.7/site-packages/middlewared/main.py:1298> exception=ModuleNotFoundError("No module named 'freenasUI.settings'")>
   Traceback (most recent call last):
   File "/usr/local/lib/python3.7/site-packages/middlewared/main.py", line 1305, in __initialize
   setup_funcs = await self.__plugins_load()
   File "/usr/local/lib/python3.7/site-packages/middlewared/main.py", line 797, in __plugins_load
   on_modules_loaded=on_modules_loaded,
   File "/usr/local/lib/python3.7/site-packages/middlewared/utils/_init_.py", line 504, in _load_plugins
   for mod in load_modules(plugins_dir):
   File "/usr/local/lib/python3.7/site-packages/middlewared/utils/_init_.py", line 463, in load_modules
   yield imp.load_module(name, fp, pathname, description)
   File "/usr/local/lib/python3.7/imp.py", line 234, in load_module
   return load_source(name, filename, file)
   File "/usr/local/lib/python3.7/imp.py", line 171, in load_source
   module = _load(spec)
   File "<frozen importlib._bootstrap>", line 696, in _load
   File "<frozen importlib._bootstrap>", line 677, in _load_unlocked
   File "<frozen importlib._bootstrap_external>", line 728, in exec_module
   File "<frozen importlib._bootstrap>", line 219, in _call_with_frames_removed
   File "/usr/local/lib/python3.7/site-packages/middlewared/plugins/datastore.py", line 15, in <module>
   django.setup()
   File "/usr/local/lib/python3.7/site-packages/django/_init_.py", line 22, in setup
   configure_logging(settings.LOGGING_CONFIG, settings.LOGGING)
   File "/usr/local/lib/python3.7/site-packages/django/conf/_init_.py", line 56, in _getattr_
   self._setup(name)
   File "/usr/local/lib/python3.7/site-packages/django/conf/_init_.py", line 41, in _setup
   self._wrapped = Settings(settings_module)
   File "/usr/local/lib/python3.7/site-packages/django/conf/_init_.py", line 110, in _init_
   mod = importlib.import_module(self.SETTINGS_MODULE)
   File "/usr/local/lib/python3.7/importlib/_init_.py", line 127, in import_module
   return _bootstrap._gcd_import(name[level:], package, level)
   File "<frozen importlib._bootstrap>", line 1006, in _gcd_import
   File "<frozen importlib._bootstrap>", line 983, in _find_and_load
   File "<frozen importlib._bootstrap>", line 965, in _find_and_load_unlocked
   ModuleNotFoundError: No module named 'freenasUI.settings'

I was able to roll back to 11.2U7 and everything seemed OK except for none of my jails iocage jails showing in the GUI (please note that warden jails still functioned properly).

iocage list gave following feedback:

1. iocage list
   Traceback (most recent call last):
   File "/usr/local/bin/iocage", line 10, in <module>
   sys.exit(cli())
   File "/usr/local/lib/python3.6/site-packages/click/core.py", line 722, in _call_
   return self.main(*args, **kwargs)
   File "/usr/local/lib/python3.6/site-packages/click/core.py", line 697, in main
   rv = self.invoke(ctx)
   File "/usr/local/lib/python3.6/site-packages/click/core.py", line 1066, in invoke
   return _process_result(sub_ctx.command.invoke(sub_ctx))
   File "/usr/local/lib/python3.6/site-packages/click/core.py", line 895, in invoke
   return ctx.invoke(self.callback, **ctx.params)
   File "/usr/local/lib/python3.6/site-packages/click/core.py", line 535, in invoke
   return callback(*args, **kwargs)
   File "/usr/local/lib/python3.6/site-packages/iocage_cli/list.py", line 81, in cli
   dataset_type, header, _long, _sort, plugin=plugins, quick=quick)
   File "/usr/local/lib/python3.6/site-packages/iocage_lib/iocage.py", line 1169, in list
   quick=quick
   File "/usr/local/lib/python3.6/site-packages/iocage_lib/ioc_list.py", line 64, in list_datasets
   iocage_lib.ioc_json.IOCJson().json_check_default_config()
   File "/usr/local/lib/python3.6/site-packages/iocage_lib/ioc_json.py", line 1924, in json_check_default_config
   self.json_write(default_props, default_json_location)
   File "/usr/local/lib/python3.6/site-packages/iocage_lib/ioc_json.py", line 431, in json_write
   if template else None
   File "libzfs.pyx", line 645, in libzfs.ZFS.get_dataset_by_path
   libzfs.ZFSException: Dataset with path not found

My /mnt/tank/iocage/defaults.json file on 11.2U7

{
"CONFIG_VERSION": "26",
"allow_chflags": 0,
"allow_mlock": 0,
"allow_mount": 0,
"allow_mount_devfs": 0,
"allow_mount_fusefs": 0,
"allow_mount_nullfs": 0,
"allow_mount_procfs": 0,
"allow_mount_tmpfs": 0,
"allow_mount_zfs": 0,
"allow_quotas": 0,
"allow_raw_sockets": 0,
"allow_set_hostname": 1,
"allow_socket_af": 0,
"allow_sysvipc": 0,
"allow_tun": 0,
"allow_vmm": 0,
"assign_localhost": 0,
"available": "readonly",
"basejail": 0,
"boot": 0,
"bpf": 0,
"children_max": "0",
"comment": "none",
"compression": "lz4",
"compressratio": "readonly",
"coredumpsize": "off",
"count": "1",
"cpuset": "off",
"cputime": "off",
"datasize": "off",
"dedup": "off",
"defaultrouter": "auto",
"defaultrouter6": "auto",
"depends": "none",
"devfs_ruleset": "4",
"dhcp": 0,
"enforce_statfs": "2",
"exec_clean": 1,
"exec_created": "/usr/bin/true",
"exec_fib": "0",
"exec_jail_user": "root",
"exec_poststart": "/usr/bin/true",
"exec_poststop": "/usr/bin/true",
"exec_prestart": "/usr/bin/true",
"exec_prestop": "/usr/bin/true",
"exec_start": "/bin/sh /etc/rc",
"exec_stop": "/bin/sh /etc/rc.shutdown",
"exec_system_jail_user": "0",
"exec_system_user": "root",
"exec_timeout": "60",
"host_domainname": "none",
"host_time": 1,
"hostid": "00000000-0000-0000-0000-0cc47a84a594",
"hostid_strict_check": 0,
"interfaces": "vnet0:bridge0",
"ip4": "new",
"ip4_addr": "none",
"ip4_saddrsel": 1,
"ip6": "new",
"ip6_addr": "none",
"ip6_saddrsel": 1,
"ip_hostname": 0,
"jail_zfs": 0,
"jail_zfs_mountpoint": "none",
"last_started": "none",
"localhost_ip": "none",
"login_flags": "-f root",
"mac_prefix": "02ff60",
"maxproc": "off",
"memorylocked": "off",
"memoryuse": "off",
"mount_devfs": 1,
"mount_fdescfs": 1,
"mount_linprocfs": 0,
"mount_procfs": 0,
"mountpoint": "readonly",
"msgqqueued": "off",
"msgqsize": "off",
"nat": 0,
"nat_backend": "ipfw",
"nat_forwards": "none",
"nat_interface": "none",
"nat_prefix": "172.16",
"nmsgq": "off",
"notes": "none",
"nsem": "off",
"nsemop": "off",
"nshm": "off",
"nthr": "off",
"openfiles": "off",
"origin": "readonly",
"owner": "root",
"pcpu": "off",
"plugin_name": "none",
"plugin_repository": "none",
"priority": "99",
"pseudoterminals": "off",
"quota": "none",
"readbps": "off",
"readiops": "off",
"reservation": "none",
"resolver": "/etc/resolv.conf",
"rlimits": "off",
"rtsold": 0,
"securelevel": "2",
"shmsize": "off",
"stacksize": "off",
"stop_timeout": "30",
"swapuse": "off",
"sync_state": "none",
"sync_target": "none",
"sync_tgt_zpool": "none",
"sysvmsg": "new",
"sysvsem": "new",
"sysvshm": "new",
"template": 0,
"type": "jail",
"used": "readonly",
"vmemoryuse": "off",
"vnet": 0,
"vnet0_mac": "none",
"vnet1_mac": "none",
"vnet2_mac": "none",
"vnet3_mac": "none",
"vnet_default_interface": "auto",
"vnet_interfaces": "none",
"wallclock": "off",
"writebps": "off",
"writeiops": "off"
}

I've attached debug log from 11.2U7