GPU cluster

24U cabinet. Four 4U GPU nodes with two 100G ports each, a dataset store, one 100G switch, two PDUs.

Devices

Dataset storeServer, 2U, 24 small drives
Fibre panel, 24 LC duplexFibre panel, 24 LC duplex
GPU node 1GPU server, 4U
GPU node 2GPU server, 4U
GPU node 3GPU server, 4U
GPU node 4GPU server, 4U
PDU, 8 outletsPDU, 8 outlets
PDU, 8 outletsPDU, 8 outlets
100G switchMikroTik CRS520-4XS-16XQ-RM

YAML

network: 1
imports:
  - ../../catalog/index.network.yaml
cables:
  cat6-blue:   { extends: cat6, color: "#3689c9" }
  cat6-green:  { extends: cat6, color: "#41a282" }
  cat6-yellow: { extends: cat6, color: "#d9a520" }
  cat6-red:    { extends: cat6, color: "#c8504a" }
racks:
  cabinet:
    type: rack-cabinet
    door: none
    units: 24
    panels:
      brush-22: { u: 22, type: brush }
      blank-1:  { u: 1, type: blank }
devices:
  spine: { model: mikrotik/crs520-4xs-16xq-rm, name: 100G switch, in: cabinet, u: 24, ports: { qsfp28-1-1: { module: qsfp28-lc-100g }, qsfp28-2-1: { module: qsfp28-lc-100g }, qsfp28-3-1: { module: qsfp28-lc-100g }, qsfp28-4-1: { module: qsfp28-lc-100g }, qsfp28-5-1: { module: qsfp28-lc-100g }, qsfp28-6-1: { module: qsfp28-lc-100g }, qsfp28-7-1: { module: qsfp28-lc-100g }, qsfp28-8-1: { module: qsfp28-lc-100g }, sfp28-1: { module: sfp28-lc-25g } } }
  fibre: { model: fibre-panel-lc-24, in: cabinet, u: 23 }
  gpu-1: { model: server-4u-gpu, name: GPU node 1, in: cabinet, u: 18, ports: { qsfp1: { module: qsfp28-lc-100g }, qsfp2: { module: qsfp28-lc-100g } } }
  gpu-2: { model: server-4u-gpu, name: GPU node 2, in: cabinet, u: 14, ports: { qsfp1: { module: qsfp28-lc-100g }, qsfp2: { module: qsfp28-lc-100g } } }
  gpu-3: { model: server-4u-gpu, name: GPU node 3, in: cabinet, u: 10, ports: { qsfp1: { module: qsfp28-lc-100g }, qsfp2: { module: qsfp28-lc-100g } } }
  gpu-4: { model: server-4u-gpu, name: GPU node 4, in: cabinet, u: 6, ports: { qsfp1: { module: qsfp28-lc-100g }, qsfp2: { module: qsfp28-lc-100g } } }
  data: { model: server-2u-24sff, name: Dataset store, in: cabinet, u: 4, ports: { sfp28-1: { module: sfp28-lc-25g } } }
  pdu-a: { model: pdu-8, in: cabinet, u: 3 }
  pdu-b: { model: pdu-8, in: cabinet, u: 2 }
links:
  gpu-1-a: { ends: [gpu-1/qsfp1, fibre/r1], cable: lc-duplex }
  gpu-1-b: { ends: [gpu-1/qsfp2, fibre/r2], cable: lc-duplex }
  s-1: { ends: [spine/qsfp28-1-1, fibre/1], cable: lc-duplex }
  s-2: { ends: [spine/qsfp28-2-1, fibre/2], cable: lc-duplex }
  gpu-2-a: { ends: [gpu-2/qsfp1, fibre/r3], cable: lc-duplex }
  gpu-2-b: { ends: [gpu-2/qsfp2, fibre/r4], cable: lc-duplex }
  s-3: { ends: [spine/qsfp28-3-1, fibre/3], cable: lc-duplex }
  s-4: { ends: [spine/qsfp28-4-1, fibre/4], cable: lc-duplex }
  gpu-3-a: { ends: [gpu-3/qsfp1, fibre/r5], cable: lc-duplex }
  gpu-3-b: { ends: [gpu-3/qsfp2, fibre/r6], cable: lc-duplex }
  s-5: { ends: [spine/qsfp28-5-1, fibre/5], cable: lc-duplex }
  s-6: { ends: [spine/qsfp28-6-1, fibre/6], cable: lc-duplex }
  gpu-4-a: { ends: [gpu-4/qsfp1, fibre/r7], cable: lc-duplex }
  gpu-4-b: { ends: [gpu-4/qsfp2, fibre/r8], cable: lc-duplex }
  s-7: { ends: [spine/qsfp28-7-1, fibre/7], cable: lc-duplex }
  s-8: { ends: [spine/qsfp28-8-1, fibre/8], cable: lc-duplex }
  data: { ends: [data/sfp28-1, fibre/r9], cable: lc-duplex }
  s-data: { ends: [spine/sfp28-1, fibre/9], cable: lc-duplex }
scenes:
  front: { title: GPU cluster, layout: [cabinet] }
  rear: { title: "GPU cluster, rear", layout: [cabinet], view: rear }