This shows you the differences between two versions of the page.
Next revision | Previous revision Next revision Both sides next revision | ||
cluster:154 [2017/03/03 19:02] hmeij07 created |
cluster:154 [2018/07/30 17:16] hmeij07 |
||
---|---|---|---|
Line 2: | Line 2: | ||
**[[cluster: | **[[cluster: | ||
- | ==== OpenHPC ==== | + | ==== OpenHPC |
- | * install vanilla CentOS 7.2 | + | * install vanilla CentOS 7.2 on // |
+ | * find Install_guide-CentOS7.2-SLURM-1.2.1-x86_64.pdf recipe guide on http:// | ||
+ | * turn selinux off | ||
+ | * next switch to iptables | ||
- | * disable firewalld, install iptables | + | < |
- | * | + | |
+ | |||
+ | [root@ohpc0-test ~]# systemctl stop NetworkManager | ||
+ | [root@ohpc0-test ~]# systemctl disable NetworkManager | ||
+ | |||
+ | [root@ohpc0-test ~]# systemctl stop firewalld | ||
+ | [root@ohpc0-test ~]# systemctl disable firewalld | ||
+ | |||
+ | [root@ohpc0-test ~]# yum install iptables-services -y | ||
+ | [root@ohpc0-test ~]# systemctl enable iptables | ||
+ | [root@ohpc0-test ~]# systemctl enable ip6tables | ||
+ | |||
+ | [root@ohpc0-test ~]# vi / | ||
+ | |||
+ | # lock up port 22: note " | ||
+ | -A INPUT -i enp8s0 -p tcp -m state --state NEW -m tcp -s 129.133.0.0/ | ||
+ | |||
+ | # local allow: note " | ||
+ | -A INPUT -i enp4s0 -d 192.168.0.0/ | ||
+ | -A INPUT -i enp4s0 -d 192.168.0.0/ | ||
+ | |||
+ | [root@ohpc0-test ~]# vi / | ||
+ | |||
+ | # comment out port 22 | ||
+ | |||
+ | [root@ohpc0-test ~]# systemctl restart iptables | ||
+ | [root@ohpc0-test ~]# systemctl restart ip6tables | ||
+ | [root@ohpc0-test ~]# iptables -L | ||
+ | Chain INPUT (policy ACCEPT) | ||
+ | target | ||
+ | ACCEPT | ||
+ | ACCEPT | ||
+ | ACCEPT | ||
+ | ACCEPT | ||
+ | ACCEPT | ||
+ | ACCEPT | ||
+ | REJECT | ||
+ | |||
+ | Chain FORWARD (policy ACCEPT) | ||
+ | target | ||
+ | REJECT | ||
+ | |||
+ | Chain OUTPUT (policy ACCEPT) | ||
+ | target | ||
+ | |||
+ | [root@ohpc0-test ~]# reboot | ||
+ | |||
+ | </ | ||
+ | |||
+ | * next add OpenHPC component, install the RPM package which also enables repo EPEL | ||
+ | |||
+ | < | ||
+ | |||
+ | [root@ohpc0-test ~]# yum install http:// | ||
+ | |||
+ | Installed: | ||
+ | ohpc-release.x86_64 0:1.2-1 | ||
+ | Dependency Installed: | ||
+ | epel-release.noarch 0:7-9 | ||
+ | |||
+ | [root@ohpc0-test ~]# yum repolist | ||
+ | repo id repo name | ||
+ | OpenHPC | ||
+ | OpenHPC-updates | ||
+ | base/ | ||
+ | *epel/ | ||
+ | extras/ | ||
+ | updates/ | ||
+ | |||
+ | </ | ||
+ | |||
+ | * Next provisioning, | ||
+ | |||
+ | < | ||
+ | |||
+ | yum -y groupinstall ohpc-base | ||
+ | yum -y groupinstall ohpc-warewulf | ||
+ | # for openlava if we decide not slurm | ||
+ | yum install tcl-devel | ||
+ | |||
+ | | ||
+ | | ||
+ | | ||
+ | |||
+ | yum -y groupinstall ohpc-slurm-server | ||
+ | |||
+ | </ | ||
+ | |||
+ | * Configure '' | ||
+ | |||
+ | ==== Configure ==== | ||
+ | |||
+ | * Warewulf (do yourself a favor and check changes in file to avoid typos) | ||
+ | |||
+ | < | ||
+ | |||
+ | perl -pi -e " | ||
+ | |||
+ | perl -pi -e " | ||
+ | |||
+ | perl -pi -e " | ||
+ | perl -pi -e " | ||
+ | |||
+ | | ||
+ | # -ni not -pi | ||
+ | perl -ni -e "print unless /^\s+Order allow, | ||
+ | |||
+ | # the recipe does not set a mysql root password but we will | ||
+ | [root@ohpc0-test]# | ||
+ | |||
+ | mysql> set password for ' | ||
+ | Query OK, 0 rows affected (0.00 sec) | ||
+ | |||
+ | [root@ohcp0-test]# | ||
+ | |||
+ | # restart/ | ||
+ | systemctl restart xinetd | ||
+ | systemctl enable mariadb.service | ||
+ | systemctl restart mariadb | ||
+ | systemctl enable httpd.service | ||
+ | systemctl restart httpd | ||
+ | |||
+ | |||
+ | </ | ||
+ | |||
+ | * Now let get ready to provision a node. First we need to build a CHROOT environment. | ||
+ | |||
+ | < | ||
+ | |||
+ | # defined repo | ||
+ | less / | ||
+ | |||
+ | # admin area | ||
+ | ls -R / | ||
+ | |||
+ | # use another disk for images | ||
+ | mkdir /data | ||
+ | mkfs.xfs -f /dev/sdb1 | ||
+ | mount /data | ||
+ | mkdir -p / | ||
+ | |||
+ | # this yields an error as 7.2.1511 is already deprecated | ||
+ | wwmkchroot centos-7 / | ||
+ | "This directory (and version of CentOS) is deprecated. | ||
+ | you should use /7/ and not /7.2.1511/ in your path." | ||
+ | # so we follow their advise and edit the repo destination s/ | ||
+ | vi / | ||
+ | |||
+ | # try again | ||
+ | wwmkchroot centos-7 / | ||
+ | # ls / | ||
+ | bin boot dev etc fastboot | ||
+ | proc root run sbin srv sys tmp usr var | ||
+ | # du -hs / | ||
+ | 490M / | ||
+ | |||
+ | </ | ||
+ | |||
+ | * Next customize the CHROOT environment | ||
+ | |||
+ | < | ||
+ | |||
+ | cp -p / | ||
+ | yum -y --installroot=/ | ||
+ | yum -y --installroot=/ | ||
+ | yum -y --installroot=/ | ||
+ | yum -y --installroot=/ | ||
+ | # pass on infiniband | ||
+ | |||
+ | # if it does not exist on **master** issue command '' | ||
+ | cat ~/ | ||
+ | cat ~/ | ||
+ | |||
+ | echo " | ||
+ | >> / | ||
+ | echo " | ||
+ | >> / | ||
+ | |||
+ | chroot / | ||
+ | echo " | ||
+ | |||
+ | # finally on **master** issue | ||
+ | perl -pi -e " | ||
+ | # this turned out to be wrong, first I change the hostname to '' | ||
+ | # added line in /etc/hosts pointing this to 192.168.1.249, | ||
+ | |||
+ | NodeName=ohpc0-slurm NodeAddr=192.168.1.249 CPUs=2 \ | ||
+ | RealMemory=8 Sockets=2 CoresPerSocket=4 ThreadsPerCore=2 State=UNKNOWN | ||
+ | NodeName=n29 NodeAddr=192.168.102.38 CPUs=2 \ | ||
+ | RealMemory=8 Sockets=2 CoresPerSocket=4 ThreadsPerCore=2 State=UNKNOWN | ||
+ | NodeName=n31 NodeAddr=192.168.102.40 CPUs=2 | ||
+ | RealMemory=8 Sockets=2 CoresPerSocket=4 ThreadsPerCore=2 State=UNKNOWN | ||
+ | PartitionName=test Nodes=n29, | ||
+ | |||
+ | |||
+ | |||
+ | |||
+ | |||
+ | echo "/home *(rw, | ||
+ | echo "/ | ||
+ | exportfs -ra | ||
+ | systemctl restart nfs | ||
+ | systemctl enable nfs-server | ||
+ | |||
+ | </ | ||
+ | |||
+ | ==== Deploy ==== | ||
+ | |||
+ | * Next we PXE boot the compute node for imaging (after building bootstrp image and VNFS) | ||
+ | |||
+ | < | ||
+ | |||
+ | # Both are loaded in database; back up if production | ||
+ | echo " | ||
+ | wwbootstrap `uname -r` | ||
+ | # Bootstrap image ' | ||
+ | wwvnfs -y --chroot / | ||
+ | # VNFS ' | ||
+ | # Wrote a new configuration file at: / | ||
+ | |||
+ | wwsh -y file import / | ||
+ | wwsh -y file set network --path / | ||
+ | wwsh -y node new ohpc0 --ipaddr=192.168.1.248 --hwaddr=00: | ||
+ | |||
+ | wwsh -y file import /etc/passwd | ||
+ | wwsh -y file import /etc/group | ||
+ | wwsh -y file import /etc/shadow | ||
+ | wwsh -y file import / | ||
+ | wwsh -y file import / | ||
+ | wwsh -y provision set ohpc0 --vnfs=centos7.2 --bootstrap=`uname -r` \ | ||
+ | | ||
+ | |||
+ | wwsh pxe update | ||
+ | wwsh dhcp update | ||
+ | systemctl restart dhcpd | ||
+ | systemctl restart httpd | ||
+ | |||
+ | </ | ||
+ | |||
+ | * Note: the next part is optional but I recommend it. Warewulf by defaults deploys '' | ||
+ | |||
+ | < | ||
+ | |||
+ | yum -y --installroot=/ | ||
+ | wwvnfs -y --chroot / | ||
+ | |||
+ | wwsh -y object modify -s bootloader=sda -t node ohpc0 | ||
+ | wwsh -y object modify -s diskpartition=sda -t node ohpc0 | ||
+ | wwsh -y object modify -s diskformat=sda1, | ||
+ | wwsh -y object modify -s filesystems=\ | ||
+ | | ||
+ | | ||
+ | wwsh -y object modify -s bootlocal=UNDEF -t node ohpc0 | ||
+ | |||
+ | wwsh pxe update | ||
+ | wwsh dhcp update | ||
+ | systemctl restart dhcpd | ||
+ | systemctl restart httpd | ||
+ | |||
+ | |||
+ | [root@ohpc0-test ~]# wwsh -y object print ohpc0 -p :all | ||
+ | #### node ohpc0 ############################################################### | ||
+ | 4: NAME = ohpc0 | ||
+ | 4: BOOTLOADER = sda | ||
+ | 4: BOOTLOCAL = UNDEF | ||
+ | 4: BOOTSTRAPID = 1 | ||
+ | 4: DISKFORMAT = sda1, | ||
+ | 4: DISKPARTITION = sda | ||
+ | 4: FILEIDS | ||
+ | 4: FILESYSTEMS = dev=sda2: | ||
+ | mountpoint=/: | ||
+ | mountpoint=/ | ||
+ | 4: NETDEVS | ||
+ | NETDEVS.enp4s0.NAME | ||
+ | NETDEVS.enp4s0.HWADDR | ||
+ | NETDEVS.enp4s0.IPADDR | ||
+ | 4: NODENAME | ||
+ | 4: VNFSID | ||
+ | |||
+ | # Strange that netmask is not listed, but when I recreated the object I used | ||
+ | wwsh -y node new ohpc0 --ipaddr=192.168.1.248 --hwaddr=00: | ||
+ | | ||
+ | |||
+ | [root@ohpc0-test ~]# wwsh -y bootstrap list | ||
+ | BOOTSTRAP NAME SIZE (M) | ||
+ | 3.10.0-327.el7.x86_64 | ||
+ | |||
+ | [root@ohpc0-test ~]# wwsh -y vnfs list | ||
+ | VNFS NAME SIZE (M) CHROOT LOCATION | ||
+ | centos7.2 | ||
+ | |||
+ | </ | ||
+ | |||
+ | Final notes. I now have a 3 node OpenHPC cluster up using CentOS 7.3.1611 ... because of the edit mention above of the provision template URL the CHROOT is at the latest version of CentOS. Thus I updated my SMS master too so that the construct bootstrap=`uname -r` builds an image compatible between SMS and CHROOT. | ||
+ | |||
+ | On towards testing the tools. | ||
+ | |||
+ | Made a little script to recreate nodes as we'll do this often. And I also some weirdo situation where eth0/1 change NIC location during PXE boot, so I work around it. | ||
+ | |||
+ | * '' | ||
+ | |||
+ | < | ||
+ | |||
+ | #!/bin/bash | ||
+ | # enable both NIC to boot from 501/500 | ||
+ | # provision black on bottom, red on top (handler swithces to this) | ||
+ | # set bootlocal to EXIT reboot, handler exits | ||
+ | # switch black to top reboot (no media, fails to hdd) | ||
+ | # insane | ||
+ | |||
+ | node=n31 | ||
+ | ipaddr0=192.168.102.40 | ||
+ | hwaddr0=1c: | ||
+ | wwsh object delete $node -y | ||
+ | wwsh node new $node --netdev=eth0 --hwaddr=$hwaddr0 --ipaddr=$ipaddr0 --netmask=255.255.0.0 | ||
+ | wwsh -y provision set $node --vnf=centos7.2 --bootstrap=`uname -r` --files=dynamic_hosts, | ||
+ | wwsh object modify -s bootloader=sda $node -y | ||
+ | wwsh object modify -s diskpartition=sda $node -y | ||
+ | wwsh object modify -s diskformat=sda1, | ||
+ | wwsh object modify -s filesystems=" | ||
+ | wwsh provision set --bootlocal=UNDEF $node -y | ||
+ | |||
+ | node=n31e | ||
+ | ipaddr0=192.168.102.40 | ||
+ | hwaddr0=1c: | ||
+ | wwsh object delete $node -y | ||
+ | wwsh node new $node --netdev=eth0 --hwaddr=$hwaddr0 --ipaddr=$ipaddr0 --netmask=255.255.0.0 | ||
+ | wwsh -y provision set $node --vnf=centos7.2 --bootstrap=`uname -r` --files=dynamic_hosts, | ||
+ | wwsh object modify -s bootloader=sda $node -y | ||
+ | wwsh object modify -s diskpartition=sda $node -y | ||
+ | wwsh object modify -s diskformat=sda1, | ||
+ | wwsh object modify -s filesystems=" | ||
+ | wwsh provision set --bootlocal=UNDEF $node -y | ||
+ | |||
+ | node=n29 | ||
+ | ipaddr0=192.168.102.38 | ||
+ | hwaddr0=1c: | ||
+ | wwsh object delete $node -y | ||
+ | wwsh node new $node --netdev=eth0 --hwaddr=$hwaddr0 --ipaddr=$ipaddr0 --netmask=255.255.0.0 | ||
+ | wwsh -y provision set $node --vnf=centos7.2 --bootstrap=`uname -r` --files=dynamic_hosts, | ||
+ | wwsh object modify -s bootloader=sda $node -y | ||
+ | wwsh object modify -s diskpartition=sda $node -y | ||
+ | wwsh object modify -s diskformat=sda1, | ||
+ | wwsh object modify -s filesystems=" | ||
+ | wwsh provision set --bootlocal=UNDEF $node -y | ||
+ | |||
+ | node=n29e | ||
+ | ipaddr0=192.168.102.38 | ||
+ | hwaddr0=1c: | ||
+ | wwsh object delete $node -y | ||
+ | wwsh node new $node --netdev=eth0 --hwaddr=$hwaddr0 --ipaddr=$ipaddr0 --netmask=255.255.0.0 | ||
+ | wwsh -y provision set $node --vnf=centos7.2 --bootstrap=`uname -r` --files=dynamic_hosts, | ||
+ | wwsh object modify -s bootloader=sda $node -y | ||
+ | wwsh object modify -s diskpartition=sda $node -y | ||
+ | wwsh object modify -s diskformat=sda1, | ||
+ | wwsh object modify -s filesystems=" | ||
+ | wwsh provision set --bootlocal=UNDEF $node -y | ||
+ | |||
+ | wwsh pxe update | ||
+ | wwsh dhcp update | ||
+ | systemctl restart dhcpd | ||
+ | systemctl restart httpd | ||
+ | echo "after first boot: wwsh provision set --bootlocal=EXIT $node" | ||
+ | |||
+ | </ | ||
+ | |||
+ | page 1 - [[cluster: | ||
\\ | \\ | ||
**[[cluster: | **[[cluster: |