Upgrade to Pro — share decks privately, control downloads, hide ads and more …

Trading Flexibility for Performance: The HPC Story in OpenStack

Trading Flexibility for Performance: The HPC Story in OpenStack

Stephen Finucane

May 27, 2019
Tweet

More Decks by Stephen Finucane

Other Decks in Technology

Transcript

  1. Trading Flexibility
    for Performance
    The HPC Story in OpenStack
    Stephen Finucane
    Software Engineer, Red Hat
    @stephenfin

    View Slide

  2. Photo by Jackson Jost on Unsplash

    View Slide

  3. $ openstack flavor create --vcpus 8 --ram 8192 --disk 0 my-flavor
    $ openstack server create --flavor my-flavor my-server

    View Slide

  4. Photo by Pero Kalimero on Unsplash

    View Slide

  5. $ openstack flavor create --vcpus 8 --ram 8192 --disk 0 my-flavor

    View Slide

  6. $ openstack flavor create --vcpus 8 --ram 8192 --disk 0 my-flavor
    $ openstack flavor set \
    --property hw:cpu_policy=dedicated \
    --property hw:cpu_thread_policy=isolate \
    --property hw:numa_nodes=2 \
    --property hw:numa_cpus.0=0-5 \
    --property hw:numa_cpus.1=6,7 \
    --property hw:numa_mem.0=6144 \
    --property hw:numa_mem.1=2048 \
    --property hw:cpu_realtime=yes \
    --property hw:cpu_realtime_mask=^0-1 \
    --property hw:mem_page_size=1GB \
    my-flavor

    View Slide

  7. $ openstack flavor create --vcpus 8 --ram 8192 --disk 0 my-flavor
    $ openstack flavor set \
    --property hw:cpu_policy=dedicated \
    --property hw:cpu_thread_policy=isolate \
    --property hw:numa_nodes=2 \
    --property hw:numa_cpus.0=0-5 \
    --property hw:numa_cpus.1=6,7 \
    --property hw:numa_mem.0=6144 \
    --property hw:numa_mem.1=2048 \
    --property hw:cpu_realtime=yes \
    --property hw:cpu_realtime_mask=^0-1 \
    --property hw:mem_page_size=1GB \
    my-flavor
    $ openstack server create --flavor my-flavor my-server

    View Slide

  8. Photo by Josep Castells on Unsplash

    View Slide

  9. Photo by Paolo Nicolello on Unsplash

    View Slide

  10. #1
    CPU Pinning

    View Slide

  11. $ openstack flavor set \
    --property hw:cpu_policy=dedicated \
    --property hw:cpu_thread_policy=isolate \
    --property hw:numa_nodes=2 \
    --property hw:numa_cpus.0=0-5 \
    --property hw:numa_cpus.1=6,7 \
    --property hw:numa_mem.0=6144 \
    --property hw:numa_mem.1=2048 \
    --property hw:cpu_realtime=yes \
    --property hw:cpu_realtime_mask=^0-1 \
    --property hw:mem_page_size=1GB \
    my-flavor

    View Slide

  12. node #0
    CPU #0 CPU #1
    CPU #2 CPU #3

    View Slide

  13. node #0
    CPU #0 CPU #1
    CPU #2 CPU #3
    $ openstack server create ...

    View Slide

  14. node #0
    CPU #0 CPU #1
    CPU #2 CPU #3
    $ openstack server create ...
    $ openstack server create ...
    $ openstack server create ...

    View Slide

  15. node #0
    CPU #0 CPU #1
    CPU #2 CPU #3
    $ openstack server create ...
    $ openstack server create ...
    $ openstack server create ...
    $ openstack server create ...
    $ openstack server create ...
    $ openstack server create ...
    $ openstack server create ...
    $ openstack server create ...
    $ openstack server create ...
    $ openstack server create ...
    $ openstack server create ...
    $ openstack server create ...
    $ openstack server create ...
    $ openstack server create ...
    $ openstack server create ...

    View Slide

  16. View Slide

  17. Is it always?

    View Slide

  18. $ openstack flavor set \
    --property hw:cpu_policy=dedicated \
    --property hw:cpu_thread_policy=isolate \
    --property hw:numa_nodes=2 \
    --property hw:numa_cpus.0=0-5 \
    --property hw:numa_cpus.1=6,7 \
    --property hw:numa_mem.0=6144 \
    --property hw:numa_mem.1=2048 \
    --property hw:cpu_realtime=yes \
    --property hw:cpu_realtime_mask=^0-1 \
    --property hw:mem_page_size=1GB \
    my-flavor

    View Slide

  19. node #0
    CPU #0 CPU #1
    CPU #2 CPU #3

    View Slide

  20. $ openstack flavor set \
    --property hw:cpu_policy=dedicated \
    --property hw:cpu_thread_policy=isolate \
    --property hw:numa_nodes=2 \
    --property hw:numa_cpus.0=0-5 \
    --property hw:numa_cpus.1=6,7 \
    --property hw:numa_mem.0=6144 \
    --property hw:numa_mem.1=2048 \
    --property hw:cpu_realtime=yes \
    --property hw:cpu_realtime_mask=^0-1 \
    --property hw:mem_page_size=1GB \
    my-flavor

    View Slide

  21. #2
    NUMA Awareness

    View Slide

  22. $ openstack flavor set \
    --property hw:cpu_policy=dedicated \
    --property hw:cpu_thread_policy=isolate \
    --property hw:numa_nodes=2 \
    --property hw:numa_cpus.0=0-5 \
    --property hw:numa_cpus.1=6,7 \
    --property hw:numa_mem.0=6144 \
    --property hw:numa_mem.1=2048 \
    --property hw:cpu_realtime=yes \
    --property hw:cpu_realtime_mask=^0-1 \
    --property hw:mem_page_size=1GB \
    my-flavor

    View Slide

  23. node #0 node #1
    node #2 node #3
    node #1
    CPU #0 CPU #1
    CPU #2 CPU #3

    View Slide

  24. node A node B
    Local Access Remote Access
    Memory Channel Interconnect Memory Channel

    View Slide

  25. node A node B
    node C node D

    View Slide

  26. node A node B
    node C node D

    View Slide

  27. node #1
    CPU #4 CPU #5
    CPU #6 CPU #7
    node #0
    CPU #0 CPU #1
    CPU #2 CPU #3

    View Slide

  28. node #1
    CPU #4 CPU #5
    CPU #6 CPU #7
    node #0
    CPU #0 CPU #1
    CPU #2 CPU #3

    View Slide

  29. node #1
    CPU #4 CPU #5
    CPU #6 CPU #7
    node #0
    CPU #0 CPU #1
    CPU #2 CPU #3

    View Slide

  30. node #1
    CPU #4 CPU #5
    CPU #6 CPU #7
    node #0
    CPU #0 CPU #1
    CPU #2 CPU #3

    View Slide

  31. $ openstack flavor set \
    --property hw:cpu_policy=dedicated \
    --property hw:cpu_thread_policy=isolate \
    --property hw:numa_nodes=2 \
    --property hw:numa_cpus.0=0-5 \
    --property hw:numa_cpus.1=6,7 \
    --property hw:numa_mem.0=6144 \
    --property hw:numa_mem.1=2048 \
    --property hw:cpu_realtime=yes \
    --property hw:cpu_realtime_mask=^0-1 \
    --property hw:mem_page_size=1GB \
    my-flavor

    View Slide

  32. node #1
    CPU #4 CPU #5
    CPU #6 CPU #7
    node #0
    CPU #0 CPU #1
    CPU #2 CPU #3

    View Slide

  33. $ openstack flavor set \
    --property hw:cpu_policy=dedicated \
    --property hw:cpu_thread_policy=isolate \
    --property hw:numa_nodes=2 \
    --property hw:numa_cpus.0=0-5 \
    --property hw:numa_cpus.1=6,7 \
    --property hw:numa_mem.0=6144 \
    --property hw:numa_mem.1=2048 \
    --property hw:cpu_realtime=yes \
    --property hw:cpu_realtime_mask=^0-1 \
    --property hw:mem_page_size=1GB \
    my-flavor

    View Slide

  34. #3
    Real Time

    View Slide

  35. Photo by Echo Grid on Unsplash

    View Slide

  36. $ openstack flavor set \
    --property hw:cpu_policy=dedicated \
    --property hw:cpu_thread_policy=isolate \
    --property hw:numa_nodes=2 \
    --property hw:numa_cpus.0=0-5 \
    --property hw:numa_cpus.1=6,7 \
    --property hw:numa_mem.0=6144 \
    --property hw:numa_mem.1=2048 \
    --property hw:cpu_realtime=yes \
    --property hw:cpu_realtime_mask=^0-1 \
    --property hw:mem_page_size=1GB \
    my-flavor

    View Slide

  37. Hardware
    Host OS/Hypervisor
    Guest OS

    View Slide

  38. 1. Disable advanced CPU features
    (HyperThreading, Power Management, Turbo Boost, …)
    2. Install host dependencies
    (real-time kernel, real-time KVM module, real-time tuned host profiles, …)
    3. Configure host memory and CPU parameters
    (enable hugepages, isolate some cores, …)
    4. Install guest dependencies
    (real-time kernel, real-time tuned guest profiles, …)

    View Slide

  39. $ openstack flavor set \
    --property hw:cpu_policy=dedicated \
    --property hw:cpu_thread_policy=isolate \
    --property hw:numa_nodes=2 \
    --property hw:numa_cpus.0=0-5 \
    --property hw:numa_cpus.1=6,7 \
    --property hw:numa_mem.0=6144 \
    --property hw:numa_mem.1=2048 \
    --property hw:cpu_realtime=yes \
    --property hw:cpu_realtime_mask=^0-1 \
    --property hw:mem_page_size=1GB \
    my-flavor

    View Slide

  40. #1
    CPU Pinning
    #2
    NUMA Awareness
    #3
    Real Time

    View Slide

  41. Photo by Paolo Nicolello on Unsplash

    View Slide

  42. NFV is complex.

    View Slide

  43. HPC is complex.

    View Slide

  44. So…?

    View Slide

  45. 1. Document it
    (make the existing stuff easier to understand)
    2. Improve it
    (extend existing features and add new ones as necessary)
    3. Replace it
    (replace the existing features with easier-to-use features)

    View Slide

  46. Photo by Marcus Dall Col on Unsplash

    View Slide

  47. Trading Flexibility
    for Performance
    The HPC Story in OpenStack
    Stephen Finucane
    Software Engineer, Red Hat
    @stephenfin

    View Slide