Nutanix Community Edition 5.18 - Installation issues and workarounds

Nutanix CE 5.18 = ce-2020.09.16-stable

Original deck is written in Japanese.(元になっている日本語版はこちら)
https://speakerdeck.com/smzksts/nutanix-ce-5-dot-18-deep-dive

This is not an official document of Nutanix Inc., Nutanix Japan G.K., or any of its affiliates.
I personally and my company are not responsible for any damage caused by referring to this material.
While the content has been written with care and consider in accuracy, it is based on validation in my personal lab environment and may not be reproducible in the other environments.
The author of this document is not a native English speaker, so there may be some inaccuracies in the English, but please read with an open mind and try to guess what the author is trying to say from the context.

Transcript

1. View Slide

2. •
   •
   •
   •
   •
   •
   •
   •
   

   View Slide

3. •
   •
   •
   •
   •
   •
   •
   

   View Slide

4. •
   •
   •
   •
   •
   •
   1. Check the IP address of Phoenix(CE Installer).
   # ip a
   2. Start sshd service on Phoenix.
   # systemctl start sshd.service
   3. SSH to Phoenix from the other client PC.
   Resize the size of SSH client window to enough size
   4. Run a command to restart the installer
   # ./ce_installer && screen -r
   5. Then it will work!
   

   View Slide

5. View Slide

6. View Slide

7. •
   •
   •
   •
   •
   •
   •
   •
   •
   

   View Slide

8. /lib/modules/4.19.84-2.el7.nutanix.20190916.276.x86_64/kernel/drivers/net
   |--bonding
   | |--bonding.ko
   |--dummy.ko
   |--ethernet
   |--agere
   | |--et131x.ko
   |--alacritech
   | |--slicoss.ko
   |--alteon
   | | |--acenic.ko
   |--altera
   | |--altera_tse.ko
   |--amazon
   | |--ena
   | | | --ena.ko
   |--amd
   | |--amd8111e.ko
   | |--pcnet32.ko
   | |--xgbe
   | | |--amd-xgbe.ko
   |--aquantia
   | |--atlantic
   | | |--atlantic.ko
   |--atheros
   | |--alx
   | | |--alx.ko
   | |--atl1c
   | | |--atl1c.ko
   | |--atl1e
   | | |--atl1e.ko
   | |--atlx
   | | |--atl1.ko
   | | |--atl2.ko
   |--broadcom
   | |--b44.ko
   | |--bcmsysport.ko
   | |--bnx2.ko
   | |--bnx2x
   | | |--bnx2x.ko
   | |--bnxt
   | | |--bnxt_en.ko
   | |--cnic.ko
   | |--genet
   | | |--genet.ko
   | |--tg3.ko
   |--brocade
   | |--bna
   | | |--bna.ko
   |--cadence
   | |--macb.ko
   |--cavium
   | |--liquidio
   | | |--liquidio.ko
   | |--thunder
   | | |--nicpf.ko
   | | |--thunder_bgx.ko
   | | |--thunder_xcv.ko
   |--chelsio
   | |--cxgb
   | |--cxgb3
   | | |--cxgb3.ko
   | |--cxgb4
   | | |--cxgb4.ko
   | |--cxgb4vf
   | | |--cxgb4vf.ko
   | | |--cxgb.ko
   | |--libcxgb
   | | |--libcxgb.ko
   |--cisco
   | |--enic
   | | |--enic.ko
   | --dlink
   | |--dl2k.ko
   | |--sundance.ko
   |--emulex
   | |--benet
   | | |--be2net.ko
   |--huawei
   | |--hinic
   | | |--hinic.ko
   |--intel
   | |--e1000
   | |--e1000e
   | | |--e1000.ko
   | | |--e1000e.ko
   | |--e100.ko
   | |--fm10k
   | | |--fm10k.ko
   | |--i40e
   | | |--i40e.ko
   | |--i40evf
   | | |--i40evf.ko
   | |--ice
   | | |--ice.ko
   | |--igb
   | | |--igb.ko
   | |--igbvf
   | | |--igbvf.ko
   | |--ixgb
   | |--ixgbe
   | | |--ixgbe.ko
   | |--ixgbevf
   | | |--ixgbevf.ko
   | | |--ixgb.ko
   |--jme.ko
   |--marvell
   | | --skge.ko
   | |--sky2.ko
   |--mellanox
   | |--mlx4
   | | |--mlx4_core.ko
   | | |--mlx4_en.ko
   | |--mlx5
   | | |--core
   | | | |--mlx5_core.ko
   |--myricom
   | |--myri10ge
   | | |--myri10ge.ko
   |--natsemi
   | |--ns83820.ko
   |--neterion
   | |--s2io.ko
   | |--vxge
   | | |--vxge.ko
   |--netronome
   | |--nfp
   | | |--nfp.ko
   |--nvidia
   | |--forcedeth.ko
   |--packetengines
   | |--hamachi.ko
   | |--yellowfin.ko
   |--qlogic
   | |--netxen
   | | |--netxen_nic.ko
   | |--qed
   | |--qede
   | | |--qede.ko
   | | |--qed.ko
   | |--qla3xxx.ko
   | |--qlcnic
   | | |--qlcnic.ko
   | | |--qlge
   | | |--qlge.ko
   |--realtek
   | |--8139cp.ko
   | |--8139too.ko
   | |--r8169.ko
   |--sfc
   | |--falcon
   | | |--sfc-falcon.ko
   | |--sfc.ko
   |--sis
   | |--sis190.ko
   |--geneve.ko
   |--hyperv
   |--hv_netvsc.ko
   |--ipvlan
   |--ipvlan.ko
   |--ipvtap.ko
   |--macvlan.ko
   |--macvtap.ko
   |--mdio.ko
   |--mii.ko
   |--netconsole.ko
   |--netdevsim
   |--netdevsim.ko
   |--net_failover.ko
   |--phy
   |--amd.ko
   |--at803x.ko
   |--bcm7xxx.ko
   |--bcm87xx.ko
   |--bcm-phy-lib.ko
   |--broadcom.ko
   |--dp83822.ko
   |--dp83tc811.ko
   |--fixed_phy.ko
   |--libphy.ko
   |--marvell.ko
   |--mdio-bcm-unimac.ko
   |--mdio-cavium.ko
   |--mdio-mscc-miim.ko
   |--mdio-thunder.ko
   |--microchip_t1.ko
   |--realtek.ko
   |--uPD60620.ko
   |--tap.ko
   |--tun.ko
   |--usb
   |--cdc_eem.ko
   |--cdc_ether.ko
   |--rndis_host.ko
   |--usbnet.ko
   |--veth.ko
   |--virtio_net.ko
   |--vmxnet3
   |--vmxnet3.ko
   |--vrf.ko
   |--vxlan.ko
   |--xen-netfront.ko
   

   View Slide

9. /lib/modules/4.19.100/kernel/drivers/net
   |--bonding
   | |--bonding.ko
   |--ethernet
   |--broadcom
   | |--bnx2x
   | | |--bnx2x.ko
   | |--bnxt
   | | |--bnxt_en.ko
   | |--tg3.ko
   |--cisco
   | |--enic
   | | |--enic.ko
   |--intel
   | |--e1000
   | |--e1000e
   | | |--e1000.ko
   | | |--e1000e.ko
   | |--igb
   | | |--igb.ko
   | |--igbvf
   | | |--igbvf.ko
   | |--ixgbe
   | | |--ixgbe.ko
   | |--ixgbevf
   | | |--ixgbevf.ko
   |--mellanox
   | |--mlx4
   | | |--mlx4_core.ko
   | | |--mlx4_en.ko
   | |--mlx5
   | | |--core
   | | | |--mlx5_core.ko
   | |--mlxfw
   | | |--mlxfw.ko
   |--hyperv
   |--hv_netvsc.ko
   |--mdio.ko
   |--mii.ko
   |--net_failover.ko
   |--usb
   |--cdc_ether.ko
   |--usbnet.ko
   |--virtio_net.ko
   |--vmxnet3
   |--vmxnet3.ko
   |--xen-netfront.ko
   

   View Slide

10. •
    •
    •
    •
    •
    •
    •
    •
    •
    •
    •
    •
    

    View Slide

11. •
    •
    •
    •
    •
    •
    →
    

    View Slide

12. # mount /dev/sdb1 /mnt/usb
    # cp /mnt/usb/linux-4.19.100.tar.gz /usr/src/kernels/
    # cd /usr/src/kernels/
    # tar xvfz linux-4.19.100.tar.gz
    # yum install -y bison flex bc elfutils-libelf-devel openssl-devel
    # cd /usr/src/kernels/linux-4.19.100
    # cp /proc/config.gz .
    # gunzip config.gz
    # mv config .config
    # make
    

    View Slide

13. # cd /usr/lib/modules/4.19.100/
    # rm -f build source
    # ln -s /usr/src/kernels/linux-4.19.100 build
    # ln -s /usr/src/kernels/linux-4.19.100 source
    

    View Slide

14. # cp /mnt/usb/r8168-8.048.03.tar.bz2 /var/tmp
    # cd /var/tmp
    # tar xvf r8168-8.048.03.tar.bz2
    # cd r8168-8.048.03
    #. /autorun.sh
    # find / -name r8168.ko
    /var/tmp/r8168-8.048.03/src/r8168.ko
    # cp /var/tmp/r8168-8.048.03/src/r8168.ko /mnt/usb
    # umount /mnt/usb
    

    View Slide

15. # mount /dev/sdd1 /mnt/usb
    # insmod /mnt/usb/r8168.ko
    # ip a
    # umount /mnt/usb
    # . /ce_installer && screen -r
    

    View Slide

16. •
    •
    •
    •
    •
    •
    •
    Source: From Hanabi's tweet
    https://twitter.com/Hanabi36610/status/1340974198705934336?s=20
    

    View Slide

17. ./ce_installer && screen -r to TUI
    

    View Slide

18. •
    •
    •
    •
    •
    •
    •
    •
    •
    •
    31 HYP_INSTALL_TIMEOUT = dict(
    32 esx=7 * 2 * 60,
    33 hyperv=15 * 2 * 60,
    34 kvm=20 * 2 * 60,
    35 xen=10 * 2 * 60)
    36 HYP_INSTALL_TIMEOUT_MAX = 30 * 60
    

    View Slide

19. •
    •
    •
    •
    •
    •
    •
    •
    

    View Slide

20. •
    •
    •
    38 #CONFIG_SWAP is not set
    ↓
    38 CONFIG_SWAP=y
    

    View Slide

21. •
    ①
    ②
    ③
    ④
    

    View Slide

22. •
    •
    •
    •
    •# swapon /mnt/iso/swapfile
    •# ./ce_installer && screen -r
    •# swapoff /mnt/iso/swapfile
    •# reboot
    

    View Slide

23. •
    •
    •
    •
    ✓
    ✓ ：
    •
    •
    

    View Slide

24. •
    •
    •
    •
    •
    •
    •
    •
    •
    •
    •
    •
    

    View Slide

25. •
    •
    •
    •
    

    View Slide

26. •
    •
    •
    • ‘
    •
    •
    •
    •
    •
    •
    

    View Slide

27. •
    •
    •
    •
    •
    [root@phoenix ~]# mount /dev/sdd5 /mnt/bootbank/
    [root@phoenix ~]# vi /mnt/bootbank/Nutanix/firstboot/first_boot_config.json
    247 use_vmfs_datastore": false,
    arrow (mark or symbol)
    247 use_vmfs_datastore": true,
    [root@phoenix ~]# umount /mnt/bootbank
    [root@phoenix ~]# reboot
    

    View Slide

28. •
    •
    •
    

    View Slide

29. •
    •
    •
    •
    •
    •
    •
    •
    

    View Slide

30. •
    •
    •
    • →
    FATAL Fatal exception encountered:
    Traceback (most recent call last):
    File "/bootbank/Nutanix/firstboot/esx_first_boot.py", line 2516, in
    main()
    File "/bootbank/Nutanix/firstboot/esx_first_boot.py", line 2361, in main
    create_svm_vmx_and_attach_rdm_disks_ce()
    File "/bootbank/Nutanix/firstboot/esx_first_boot.py", line 1478, in
    create_svm_vmx_and_attach_rdm_disks_ce
    dev_paths = get_disk_locations_ce()
    File "/bootbank/Nutanix/firstboot/esx_first_boot.py", line 1019, in get_disk_locations_ce
    device_identifiers.remove(disk[:-3])
    ValueError: list.remove(x): x not in list
    

    View Slide

31. /bootbank/Nutanix/firstboot/esx_first_boot.py, around line 1019
    def get_disk_locations_ce():
    disk_dict = {}
    device_identifiers = [].
    location = 1
    out = run_cmd(["ls /vmfs/devices/disks"])
    for disk in out.splitlines():
    if (disk.startswith("vml")):
    continue
    if (":" not in disk):
    for serial in P_LIST.ce_serials:
    if serial in disk:
    device_identifiers.append(disk)
    disk_dict[disk] = location
    location += 1
    for wwn in P_LIST.ce_wwns:
    if wwn in disk:
    device_identifiers.append(disk)
    disk_dict[disk] = location
    location += 1
    # We remove esxi host disk from disk available to CVM
    elif disk[-3:] == ":10":
    print(disk[:-3])
    device_identifiers.remove(disk[:-3])
    disk_dict.pop(disk[:-3])
    location -= 1
    [root@Failed-Install:~] ls /vmfs/devices/disks/
    mpx.vmhba32:C0:T0:L0
    mpx.vmhba32:C0:T0:L0:1
    mpx.vmhba32:C0:T0:L0:10
    mpx.vmhba32:C0:T0:L0:2
    mpx.vmhba32:C0:T0:L0:5
    mpx.vmhba32:C0:T0:L0:6
    mpx.vmhba32:C0:T0:L0:7
    mpx.vmhba32:C0:T0:L0:8
    mpx.vmhba32:C0:T0:L0:9
    t10.ATA_____ Crucial_CT525MX300SSD4 __________________________163413C75247
    t10.ATA_____ Crucial_CT525MX300SSD4
    __________________________163413C75247:1
    t10.ATA_____ Crucial_CT525MX300SSD4
    __________________________163413C75247:2
    t10.ATA_____ Crucial_CT525MX300SSD4
    __________________________163413C75247:3
    t10.ATA_____ Crucial_CT525MX300SSD4
    __________________________163413C75247:4
    t10.ATA_____ Crucial_CT525MX300SSD4 __________________________163413C75345
    t10.ATA_____ Crucial_CT525MX300SSD4
    __________________________163413C75345:1
    vml.000100000220022002202313632343133433735323437437275636961
    vml.000100000220022002231336333343133433735323437437275636961:1
    vml.000100000220022002231336333343133433735323437437275636961:2
    vml.00010000022002200220231336333343133433735323437437275636961:3
    vml.000100000220022002231336333343133433735323437437275636961:4
    vml.000100000220022002202313632343133433735333435437275636961
    vml.00010000022002200223133633333433735333435437275636961:1
    vml.0100000000303754304c4f374d45363834414134374d6173732053
    vml.0100000000303754304c4f374d45363834414134374d6173732053:1
    vml.0100000000303754304c4f374d45363834414134374d6173732053:10
    vml.0100000000303754304c4f374d45363834414134374d6173732053:2
    vml.0100000000303754304c4f374d45363834414134374d6173732053:5
    vml.0100000000303754304c4f374d45363834414134374d6173732053:6
    vml.0100000000303754304c4f374d45363834414134374d6173732053:7
    vml.0100000000303754304c4f374d45363834414134374d6173732053:8
    vml.0100000000303754304c4f374d45363834414134374d6173732053:9
    •
    •
    •
    •
    

    View Slide

32. for wwn in P_LIST.ce_wwns:
    if wwn in disk:
    device_identifiers.append(disk)
    disk_dict[disk] = location
    location += 1
    # We remove esxi host disk from disk available to CVM
    elif disk[-3:] == ":10":
    print(disk[:-3])
    device_identifiers.remove(disk[:-3])
    disk_dict.pop(disk[:-3])
    location -= 1
    for wwn in P_LIST.ce_wwns:
    if (wwn != None and wwn in disk) and (disk not in device_identifiers):
    # if wwn in disk:
    device_identifiers.append(disk)
    disk_dict[disk] = location
    location += 1
    # We remove esxi host disk from disk available to CVM
    elif disk[-3:] == ":10":
    print(disk[:-3])
    if disk[:-3] in device_identifiers:
    device_identifiers.remove(disk[:-3])
    disk_dict.pop(disk[:-3])
    location -= 1
    

    View Slide

33. •
    •
    [root@Failed-Install:~] python /bootbank/Nutanix/firstboot/esx_first_boot.py
    [root@Failed-Install:~] rm /bootbank/Nutanix/firstboot/.firstboot_fail
    

    View Slide

34. •
    •
    •
    •
    •
    •
    •
    

    View Slide

35. View Slide