Nutanix Community Edition 5.18 - Installation issues and workarounds

Nutanix CE 5.18 = ce-2020.09.16-stable

Original deck is written in Japanese.(元になっている日本語版はこちら)
https://speakerdeck.com/smzksts/nutanix-ce-5-dot-18-deep-dive

This is not an official document of Nutanix Inc., Nutanix Japan G.K., or any of its affiliates.
I personally and my company are not responsible for any damage caused by referring to this material.
While the content has been written with care and consider in accuracy, it is based on validation in my personal lab environment and may not be reproducible in the other environments.
The author of this document is not a native English speaker, so there may be some inaccuracies in the English, but please read with an open mind and try to guess what the author is trying to say from the context.

Transcript

2. • • • • • • • •
3. • • • • • • •
4. • • • • • • 1. Check the IP address of Phoenix(CE Installer). # ip a 2. Start sshd service on Phoenix. # systemctl start sshd.service 3. SSH to Phoenix from the other client PC. Resize the size of SSH client window to enough size 4. Run a command to restart the installer # ./ce_installer && screen -r 5. Then it will work!
7. • • • • • • • • •
8. /lib/modules/4.19.84-2.el7.nutanix.20190916.276.x86_64/kernel/drivers/net |--bonding | |--bonding.ko |--dummy.ko |--ethernet |--agere | |--et131x.ko |--alacritech | |--slicoss.ko |--alteon | | |--acenic.ko |--altera | |--altera_tse.ko |--amazon | |--ena | | | --ena.ko |--amd | |--amd8111e.ko | |--pcnet32.ko | |--xgbe | | |--amd-xgbe.ko |--aquantia | |--atlantic | | |--atlantic.ko |--atheros | |--alx | | |--alx.ko | |--atl1c | | |--atl1c.ko | |--atl1e | | |--atl1e.ko | |--atlx | | |--atl1.ko | | |--atl2.ko |--broadcom | |--b44.ko | |--bcmsysport.ko | |--bnx2.ko | |--bnx2x | | |--bnx2x.ko | |--bnxt | | |--bnxt_en.ko | |--cnic.ko | |--genet | | |--genet.ko | |--tg3.ko |--brocade | |--bna | | |--bna.ko |--cadence | |--macb.ko |--cavium | |--liquidio | | |--liquidio.ko | |--thunder | | |--nicpf.ko | | |--thunder_bgx.ko | | |--thunder_xcv.ko |--chelsio | |--cxgb | |--cxgb3 | | |--cxgb3.ko | |--cxgb4 | | |--cxgb4.ko | |--cxgb4vf | | |--cxgb4vf.ko | | |--cxgb.ko | |--libcxgb | | |--libcxgb.ko |--cisco | |--enic | | |--enic.ko | --dlink | |--dl2k.ko | |--sundance.ko |--emulex | |--benet | | |--be2net.ko |--huawei | |--hinic | | |--hinic.ko |--intel | |--e1000 | |--e1000e | | |--e1000.ko | | |--e1000e.ko | |--e100.ko | |--fm10k | | |--fm10k.ko | |--i40e | | |--i40e.ko | |--i40evf | | |--i40evf.ko | |--ice | | |--ice.ko | |--igb | | |--igb.ko | |--igbvf | | |--igbvf.ko | |--ixgb | |--ixgbe | | |--ixgbe.ko | |--ixgbevf | | |--ixgbevf.ko | | |--ixgb.ko |--jme.ko |--marvell | | --skge.ko | |--sky2.ko |--mellanox | |--mlx4 | | |--mlx4_core.ko | | |--mlx4_en.ko | |--mlx5 | | |--core | | | |--mlx5_core.ko |--myricom | |--myri10ge | | |--myri10ge.ko |--natsemi | |--ns83820.ko |--neterion | |--s2io.ko | |--vxge | | |--vxge.ko |--netronome | |--nfp | | |--nfp.ko |--nvidia | |--forcedeth.ko |--packetengines | |--hamachi.ko | |--yellowfin.ko |--qlogic | |--netxen | | |--netxen_nic.ko | |--qed | |--qede | | |--qede.ko | | |--qed.ko | |--qla3xxx.ko | |--qlcnic | | |--qlcnic.ko | | |--qlge | | |--qlge.ko |--realtek | |--8139cp.ko | |--8139too.ko | |--r8169.ko |--sfc | |--falcon | | |--sfc-falcon.ko | |--sfc.ko |--sis | |--sis190.ko |--geneve.ko |--hyperv |--hv_netvsc.ko |--ipvlan |--ipvlan.ko |--ipvtap.ko |--macvlan.ko |--macvtap.ko |--mdio.ko |--mii.ko |--netconsole.ko |--netdevsim |--netdevsim.ko |--net_failover.ko |--phy |--amd.ko |--at803x.ko |--bcm7xxx.ko |--bcm87xx.ko |--bcm-phy-lib.ko |--broadcom.ko |--dp83822.ko |--dp83tc811.ko |--fixed_phy.ko |--libphy.ko |--marvell.ko |--mdio-bcm-unimac.ko |--mdio-cavium.ko |--mdio-mscc-miim.ko |--mdio-thunder.ko |--microchip_t1.ko |--realtek.ko |--uPD60620.ko |--tap.ko |--tun.ko |--usb |--cdc_eem.ko |--cdc_ether.ko |--rndis_host.ko |--usbnet.ko |--veth.ko |--virtio_net.ko |--vmxnet3 |--vmxnet3.ko |--vrf.ko |--vxlan.ko |--xen-netfront.ko
9. /lib/modules/4.19.100/kernel/drivers/net |--bonding | |--bonding.ko |--ethernet |--broadcom | |--bnx2x | | |--bnx2x.ko | |--bnxt | | |--bnxt_en.ko | |--tg3.ko |--cisco | |--enic | | |--enic.ko |--intel | |--e1000 | |--e1000e | | |--e1000.ko | | |--e1000e.ko | |--igb | | |--igb.ko | |--igbvf | | |--igbvf.ko | |--ixgbe | | |--ixgbe.ko | |--ixgbevf | | |--ixgbevf.ko |--mellanox | |--mlx4 | | |--mlx4_core.ko | | |--mlx4_en.ko | |--mlx5 | | |--core | | | |--mlx5_core.ko | |--mlxfw | | |--mlxfw.ko |--hyperv |--hv_netvsc.ko |--mdio.ko |--mii.ko |--net_failover.ko |--usb |--cdc_ether.ko |--usbnet.ko |--virtio_net.ko |--vmxnet3 |--vmxnet3.ko |--xen-netfront.ko
10. • • • • • • • • • • • •
11. • • • • • • →
12. # mount /dev/sdb1 /mnt/usb # cp /mnt/usb/linux-4.19.100.tar.gz /usr/src/kernels/ # cd /usr/src/kernels/ # tar xvfz linux-4.19.100.tar.gz # yum install -y bison flex bc elfutils-libelf-devel openssl-devel # cd /usr/src/kernels/linux-4.19.100 # cp /proc/config.gz . # gunzip config.gz # mv config .config # make
13. # cd /usr/lib/modules/4.19.100/ # rm -f build source # ln -s /usr/src/kernels/linux-4.19.100 build # ln -s /usr/src/kernels/linux-4.19.100 source
14. # cp /mnt/usb/r8168-8.048.03.tar.bz2 /var/tmp # cd /var/tmp # tar xvf r8168-8.048.03.tar.bz2 # cd r8168-8.048.03 #. /autorun.sh # find / -name r8168.ko /var/tmp/r8168-8.048.03/src/r8168.ko # cp /var/tmp/r8168-8.048.03/src/r8168.ko /mnt/usb # umount /mnt/usb
15. # mount /dev/sdd1 /mnt/usb # insmod /mnt/usb/r8168.ko # ip a # umount /mnt/usb # . /ce_installer && screen -r
16. • • • • • • • Source: From Hanabi's tweet https://twitter.com/Hanabi36610/status/1340974198705934336?s=20
17. ./ce_installer && screen -r to TUI
18. • • • • • • • • • • 31 HYP_INSTALL_TIMEOUT = dict( 32 esx=7 * 2 * 60, 33 hyperv=15 * 2 * 60, 34 kvm=20 * 2 * 60, 35 xen=10 * 2 * 60) 36 HYP_INSTALL_TIMEOUT_MAX = 30 * 60
19. • • • • • • • •
20. • • • 38 #CONFIG_SWAP is not set ↓ 38 CONFIG_SWAP=y
21. • ① ② ③ ④
22. • • • • •# swapon /mnt/iso/swapfile •# ./ce_installer && screen -r •# swapoff /mnt/iso/swapfile •# reboot
23. • • • • ✓ ✓ ： • •
24. • • • • • • • • • • • •
25. • • • •
26. • • • • ‘ • • • • • •
27. • • • • • [root@phoenix ~]# mount /dev/sdd5 /mnt/bootbank/ [root@phoenix ~]# vi /mnt/bootbank/Nutanix/firstboot/first_boot_config.json 247 use_vmfs_datastore": false, arrow (mark or symbol) 247 use_vmfs_datastore": true, [root@phoenix ~]# umount /mnt/bootbank [root@phoenix ~]# reboot
28. • • •
29. • • • • • • • •
30. • • • • → FATAL Fatal exception encountered: Traceback (most recent call last): File "/bootbank/Nutanix/firstboot/esx_first_boot.py", line 2516, in <module> main() File "/bootbank/Nutanix/firstboot/esx_first_boot.py", line 2361, in main create_svm_vmx_and_attach_rdm_disks_ce() File "/bootbank/Nutanix/firstboot/esx_first_boot.py", line 1478, in create_svm_vmx_and_attach_rdm_disks_ce dev_paths = get_disk_locations_ce() File "/bootbank/Nutanix/firstboot/esx_first_boot.py", line 1019, in get_disk_locations_ce device_identifiers.remove(disk[:-3]) ValueError: list.remove(x): x not in list
31. /bootbank/Nutanix/firstboot/esx_first_boot.py, around line 1019 def get_disk_locations_ce(): disk_dict = {} device_identifiers = []. location = 1 out = run_cmd(["ls /vmfs/devices/disks"]) for disk in out.splitlines(): if (disk.startswith("vml")): continue if (":" not in disk): for serial in P_LIST.ce_serials: if serial in disk: device_identifiers.append(disk) disk_dict[disk] = location location += 1 for wwn in P_LIST.ce_wwns: if wwn in disk: device_identifiers.append(disk) disk_dict[disk] = location location += 1 # We remove esxi host disk from disk available to CVM elif disk[-3:] == ":10": print(disk[:-3]) device_identifiers.remove(disk[:-3]) disk_dict.pop(disk[:-3]) location -= 1 [root@Failed-Install:~] ls /vmfs/devices/disks/ mpx.vmhba32:C0:T0:L0 mpx.vmhba32:C0:T0:L0:1 mpx.vmhba32:C0:T0:L0:10 mpx.vmhba32:C0:T0:L0:2 mpx.vmhba32:C0:T0:L0:5 mpx.vmhba32:C0:T0:L0:6 mpx.vmhba32:C0:T0:L0:7 mpx.vmhba32:C0:T0:L0:8 mpx.vmhba32:C0:T0:L0:9 t10.ATA_____ Crucial_CT525MX300SSD4 __________________________163413C75247 t10.ATA_____ Crucial_CT525MX300SSD4 __________________________163413C75247:1 t10.ATA_____ Crucial_CT525MX300SSD4 __________________________163413C75247:2 t10.ATA_____ Crucial_CT525MX300SSD4 __________________________163413C75247:3 t10.ATA_____ Crucial_CT525MX300SSD4 __________________________163413C75247:4 t10.ATA_____ Crucial_CT525MX300SSD4 __________________________163413C75345 t10.ATA_____ Crucial_CT525MX300SSD4 __________________________163413C75345:1 vml.000100000220022002202313632343133433735323437437275636961 vml.000100000220022002231336333343133433735323437437275636961:1 vml.000100000220022002231336333343133433735323437437275636961:2 vml.00010000022002200220231336333343133433735323437437275636961:3 vml.000100000220022002231336333343133433735323437437275636961:4 vml.000100000220022002202313632343133433735333435437275636961 vml.00010000022002200223133633333433735333435437275636961:1 vml.0100000000303754304c4f374d45363834414134374d6173732053 vml.0100000000303754304c4f374d45363834414134374d6173732053:1 vml.0100000000303754304c4f374d45363834414134374d6173732053:10 vml.0100000000303754304c4f374d45363834414134374d6173732053:2 vml.0100000000303754304c4f374d45363834414134374d6173732053:5 vml.0100000000303754304c4f374d45363834414134374d6173732053:6 vml.0100000000303754304c4f374d45363834414134374d6173732053:7 vml.0100000000303754304c4f374d45363834414134374d6173732053:8 vml.0100000000303754304c4f374d45363834414134374d6173732053:9 • • • •
32. for wwn in P_LIST.ce_wwns: if wwn in disk: device_identifiers.append(disk) disk_dict[disk] = location location += 1 # We remove esxi host disk from disk available to CVM elif disk[-3:] == ":10": print(disk[:-3]) device_identifiers.remove(disk[:-3]) disk_dict.pop(disk[:-3]) location -= 1 for wwn in P_LIST.ce_wwns: if (wwn != None and wwn in disk) and (disk not in device_identifiers): # if wwn in disk: device_identifiers.append(disk) disk_dict[disk] = location location += 1 # We remove esxi host disk from disk available to CVM elif disk[-3:] == ":10": print(disk[:-3]) if disk[:-3] in device_identifiers: device_identifiers.remove(disk[:-3]) disk_dict.pop(disk[:-3]) location -= 1
33. • • [root@Failed-Install:~] python /bootbank/Nutanix/firstboot/esx_first_boot.py [root@Failed-Install:~] rm /bootbank/Nutanix/firstboot/.firstboot_fail
34. • • • • • • •