这是本人在虚拟机上搭建GPFS的步骤，安装包自备，可能有少少错误，请见谅

172.16.40.131 node1172.16.40.132 node2172.16.40.133node3172.16.40.134node4 add some additional disks to node4A, basic setting (do this setting on each nodes)1, /etc/hosts on all nodes 172.16.40.131node1 172.16.40.132 node2 172.16.40.133 node3 172.16.40.134 node42, disableselinux by /etc/sysconfig/selinux3, disable iptables chkconfig iptables off4, make ssh access to other nodes (includeitself) without password by root ssh-keygen -t rsa -b 2048 for srv innode{1,2,3,4}; do ssh-copy-id -i .ssh/id_rsa.pub $srv;done justfor test: for srv in node{1,2,3,4}; dossh $srv uname -r;done5, set server start level as 3 [root@localhost~]# grep id: /etc/inittab Id:3:initdefault: B, setup yum server (node4)1, install vsftp: rpm -ivh vsftpd-2.0.5-16.el5_4.1.i386.rpm2, copy packages: [root@node4 RHEL_5.5 i386 DVD]# cp -raCluster ClusterStorage Server VT /var/ftp/pub/3, [root@node4Server]# cat /etc/yum.repos.d/local.repo[Server]name=serverbaseurl=ftp://localhost/pub/Servergpgcheck=1gpgkey=file:///etc/pki/rpm-gpg/RPM-GPG-KEY-redhat-release [Cluster]name=clusterbaseurl=ftp://localhost/pub/Clustergpgcheck=1gpgkey=file:///etc/pki/rpm-gpg/RPM-GPG-KEY-redhat-release [ClusterStorage]name=serverbaseurl=ftp://localhost/pub/ClusterStoragegpgcheck=1gpgkey=file:///etc/pki/rpm-gpg/RPM-GPG-KEY-redhat-release [VT]name=VTbaseurl=ftp://localhost/pub/VTgpgcheck=1gpgkey=file:///etc/pki/rpm-gpg/RPM-GPG-KEY-redhat-release 4, rpm --import/etc/pki/rpm-gpg/RPM-GPG-KEY-redhat-release 5, [root@node4 Server]# rpm -ivhcreaterepo-0.4.11-3.el5.noarch.rpmPreparing... ########################################### [100%] 1:createrepo ########################################### [100%][root@node3 Server]#6, [size=7.5pt]createrepo -g/var/ftp/pub/Server/repodata/comps-rhel5-server-core.xml /var/ftp/pub/Server/[size=7.5pt]createrepo -g/var/ftp/pub/Cluster/repodata/comps-rhel5-cluster.xml /var/ftp/pub/Cluster/ [size=7.5pt]createrepo -g/var/ftp/pub/ClusterStorage/repodata/comps-rhel5-cluster-st.xml /var/ftp/pub/ClusterStorage/[size=7.5pt]createrepo -g/var/ftp/pub/VT/repodata/comps-rhel5-vt.xml /var/ftp/pub/VT/[size=7.5pt] [size=7.5pt]7 [size=7.5pt][root@node3 pub]#service vsftpd start[size=7.5pt]Starting vsftpd forvsftpd: [ OK ][size=7.5pt] C, setting ISCSI[size=7.5pt]1, Install scsi-target-utils-0.0-6.20091205snap.el5_4.1.i386.rpm[size=7.5pt][root@node4Server]# yum installscsi-target-utils-0.0-6.20091205snap.el5_4.1.i386.rpm[size=7.5pt]2, add belowsetting into /etc/tgt/targets.conf[i][size=7.5pt][i][size=7.5pt] backing-store/dev/sdc[i][size=7.5pt] initiator-address172.16.40.0/24[i][size=7.5pt][i][size=7.5pt] [i][size=7.5pt][root@node4 ClusterStorage]# chkconfig tgtd on[i][size=7.5pt][root@node4 ClusterStorage]# service tgtd start[size=7.5pt] [size=7.5pt]3, [root@node4 ~]#tgt-admin --show[size=7.5pt]Target 1:iqn.2012-07.com.node3.www:target1[size=7.5pt] System information:[size=7.5pt] Driver: iscsi[size=7.5pt] State: ready[size=7.5pt] [size=7.5pt]I_T nexus information:[size=7.5pt] LUN information:[size=7.5pt] [size=7.5pt]LUN: 0[size=7.5pt] Type: controller[size=7.5pt] SCSI ID: IET 00010000[size=7.5pt] SCSI SN: beaf10[size=7.5pt] Size: 0 MB[size=7.5pt] Online: Yes[size=7.5pt] Removable media: No[size=7.5pt] Backing store type: rdwr[size=7.5pt] Backing store path: None[size=7.5pt] LUN: 1[size=7.5pt] Type: disk[size=7.5pt] SCSI ID: IET 00010001[size=7.5pt] SCSI SN: beaf11[size=7.5pt] Size: 42950 MB[size=7.5pt] Online: Yes[size=7.5pt] Removable media: No[size=7.5pt] Backing store type: rdwr[size=7.5pt] Backing store path: /dev/sdc[size=7.5pt] Account information:[size=7.5pt] ACL information:[size=7.5pt] 172.16.40.0/24[size=7.5pt][root@node4~]#[size=7.5pt] [size=7.5pt]4, On client side:(node1, node2, node3)[size=7.5pt][root@node1 ~]# rpm -ivh iscsi-initiator-utils-6.2.0.872-16.el5.i386.rpm[size=7.5pt]warning:iscsi-initiator-utils-6.2.0.872-16.el5.i386.rpm:Header V3 DSA signature: NOKEY, key ID 37017186[size=7.5pt]Preparing... ########################################### [100%][size=7.5pt] 1:iscsi-initiator-utils ###########################################[100%][size=7.5pt] [size=7.5pt][root@node1 ~]# service iscsi start[size=7.5pt]iscsid is stopped[size=7.5pt]Starting iSCSIdaemon: [ OK ][size=7.5pt] [ OK ][size=7.5pt]Setting up iSCSItargets: iscsiadm: No records found[size=7.5pt] [ OK ][size=7.5pt][root@node1 ~]#[size=7.5pt] [size=7.5pt][root@node1 ~]# iscsiadm -m discovery -t sendtargets -p172.16.40.134:3260[size=7.5pt]172.16.40.143:3260,1iqn.2012-07.com.node4.www:target1[size=7.5pt][root@node1 ~]#[size=7.5pt] [size=7.5pt][root@node1~]# iscsiadm -m node -Tiqn.2012-07.com.node3.www:target1 -l[size=7.5pt]Logging in to[iface: default, target: iqn.2012-07.com.node4.www:target1, portal:172.16.40.143,3260] (multiple)[size=7.5pt]Login to [iface:default, target: iqn.2012-07.com.node4.www:target1, portal: 172.16.40.143,3260] successful.[size=7.5pt] [size=7.5pt][root@node1 ~]#fdisk -l /dev/sdc[size=7.5pt]Disk /dev/sdc: 10.7GB, 10737418240 bytes[size=7.5pt]255 heads, 63sectors/track, 1305 cylinders[size=7.5pt]Units = cylindersof 16065 * 512 = 8225280 bytes[size=7.5pt] [size=7.5pt]Disk /dev/sdcdoesn't contain a valid partition table[size=7.5pt][root@node1 ~]#[size=7.5pt] [size=7.5pt] D: Install GPFS packages[size=7.5pt][root@node1x86_32]# ls[size=7.5pt]gpfs.base-3.3.0-5.i386.rpm gpfs.gpl-3.3.0-5.noarch.rpm gpfs.libsrc-3.3.0-5.noarch.rpm gpfs.src-3.3.0-5.noarch.rpm[size=7.5pt]gpfs.docs-3.3.0-5.noarch.rpm gpfs.gui-3.3.0-5.i386.rpm gpfs.msg.en_US-3.3.0-5.noarch.rpm[size=7.5pt][root@node1x86_32]# rpm -ivh *.rpm[size=7.5pt]error: Faileddependencies:[size=7.5pt] libstdc++.so.5 is needed by gpfs.base-3.3.0-5.i386[size=7.5pt] libstdc++.so.5(GLIBCPP_3.2) is neededby gpfs.base-3.3.0-5.i386[size=7.5pt] libstdc++.so.5(GLIBCPP_3.2.2)is needed by gpfs.base-3.3.0-5.i386[size=7.5pt][root@node1x86_32]#[size=7.5pt][root@node1 ~]# rpm -ivh compat-libstdc++-33-3.2.3-61.i386.rpm[size=7.5pt]warning:compat-libstdc++-33-3.2.3-61.i386.rpm:Header V3 DSA signature: NOKEY, key ID 37017186[size=7.5pt]Preparing... ###########################################[100%][size=7.5pt] 1:compat-libstdc++-33 ###########################################[100%][size=7.5pt][root@node1 ~]#[size=7.5pt][root@node1x86_32]# ls[size=7.5pt]gpfs.base-3.3.0-5.i386.rpm gpfs.gpl-3.3.0-5.noarch.rpm gpfs.libsrc-3.3.0-5.noarch.rpm gpfs.src-3.3.0-5.noarch.rpm[size=7.5pt]gpfs.docs-3.3.0-5.noarch.rpm gpfs.gui-3.3.0-5.i386.rpm gpfs.msg.en_US-3.3.0-5.noarch.rpm[size=7.5pt][root@node1x86_32]# rpm -ivh *[size=7.5pt]Preparing... ########################################### [100%][size=7.5pt] 1:gpfs.src ########################################### [ 14%][size=7.5pt] 2:gpfs.base ########################################### [ 29%][size=7.5pt] 3:gpfs.docs ########################################### [ 43%][size=7.5pt] 4:gpfs.gpl ###########################################[ 57%][size=7.5pt] 5:gpfs.gui ########################################### [ 71%][size=7.5pt]You may start theGPFS GUI now by typing : /etc/init.d/gpfsgui start[size=7.5pt]Alternatively, theGPFS GUI will start on reboot.[size=7.5pt] 6:gpfs.libsrc ########################################### [ 86%][size=7.5pt] 7:gpfs.msg.en_US ########################################### [100%][size=7.5pt][root@node1x86_32]#[size=7.5pt] [size=7.5pt][root@node1x86_32]# cd /usr/lpp/mmfs/src/[size=7.5pt][root@node1 src]# make Autoconfig[size=7.5pt]cd /usr/lpp/mmfs/src/config;./configure --genenvonly; /usr/bin/cpp -P def.mk.proto > ./def.mk; exit $?|| exit 1[size=7.5pt][root@node1 src]# make World[size=7.5pt]Verifying thattools to build the portability layer exist....[size=7.5pt]cpp present[size=7.5pt]gcc present[size=7.5pt]g++ present[size=7.5pt]ld present[size=7.5pt]cd /usr/lpp/mmfs/src/config;/usr/bin/cpp -P def.mk.proto > ./def.mk; exit $? || exit 1[size=7.5pt]rm -rf/usr/lpp/mmfs/src/include /usr/lpp/mmfs/src/bin /usr/lpp/mmfs/src/lib[size=7.5pt]mkdir/usr/lpp/mmfs/src/include /usr/lpp/mmfs/src/bin /usr/lpp/mmfs/src/lib[size=7.5pt]rm -f//usr/lpp/mmfs/src/gpl-linux/gpl_kernel.tmp.ver[size=7.5pt]cleaning(/usr/lpp/mmfs/src/ibm-kxi)[size=7.5pt]make[1]: Enteringdirectory `/usr/lpp/mmfs/src/ibm-kxi'[size=7.5pt]......[size=7.5pt] [size=7.5pt][root@node1 src]# make InstallImages[size=7.5pt](cd gpl-linux;/usr/bin/make InstallImages; \[size=7.5pt] exit $?) || exit 1[size=7.5pt]make[1]: Entering directory`/usr/lpp/mmfs/src/gpl-linux'[size=7.5pt]Pre-kbuild step1...[size=7.5pt]make[2]: Enteringdirectory `/usr/src/kernels/2.6.18-194.el5-i686'[size=7.5pt] INSTALL/usr/lpp/mmfs/src/gpl-linux/kdump-kern-dummy.ko[size=7.5pt] INSTALL/usr/lpp/mmfs/src/gpl-linux/kdump-kern-dwarfs.ko[size=7.5pt] INSTALL /usr/lpp/mmfs/src/gpl-linux/mmfs26.ko[size=7.5pt] INSTALL/usr/lpp/mmfs/src/gpl-linux/mmfslinux.ko[size=7.5pt] INSTALL/usr/lpp/mmfs/src/gpl-linux/tracedev.ko[size=7.5pt] DEPMOD 2.6.18-194.el5[size=7.5pt]make[2]: Leavingdirectory `/usr/src/kernels/2.6.18-194.el5-i686'[size=7.5pt]make[1]: Leavingdirectory `/usr/lpp/mmfs/src/gpl-linux'[size=7.5pt][root@node1 src]#[size=7.5pt][root@node2 ~]# export PATH=$PATH:/usr/lpp/mmfs/bin[size=7.5pt][root@node2 ~]# echo 'export PATH=$PATH:/usr/lpp/mmfs/bin'>> /etc/profile[size=7.5pt][root@node2 ~]#[size=7.5pt] [size=7.5pt] [size=7.5pt]reboot all nodes after GPFS packagesinstalledE, create cluster[size=7.5pt][root@node1 ~]# vi nodelist[size=7.5pt][root@node1 ~]# cat nodelist[size=7.5pt]node1:quorum-manager[size=7.5pt]node2:quorum-manager[size=7.5pt][root@node1 ~]#[size=7.5pt][root@node1 ~]# mmcrcluster -N nodelist -p node1 -s node2-r /usr/bin/ssh -R /usr/bin/scp -C Test_GPFS_Cluster[size=7.5pt]Sun Nov 24 17:04:01PST 2013: mmcrcluster: Processing node node1[size=7.5pt]Sun Nov 24 17:04:01PST 2013: mmcrcluster: Processing node node2[size=7.5pt]mmcrcluster:Command successfully completed[size=7.5pt]mmcrcluster:Warning: Not all nodes have proper GPFS license designations.[size=7.5pt] Use the mmchlicense command to designatelicenses as needed.[size=7.5pt]mmcrcluster:Propagating the cluster configuration data to all[size=7.5pt] affected nodes. This is an asynchronous process.[size=7.5pt][root@node1~]# mmchlicense server --accept -Nnode1,node2[size=7.5pt] [size=7.5pt]The following nodeswill be designated as possessing GPFS server licenses:[size=7.5pt] node1[size=7.5pt] node2[size=7.5pt]mmchlicense:Command successfully completed[size=7.5pt]mmchlicense:Propagating the cluster configuration data to all[size=7.5pt] affected nodes. This is an asynchronous process.[size=7.5pt][root@node1 ~]#F, create NSD disks[size=7.5pt][root@node1 ~]# vi nsdlist[size=7.5pt][root@node1 ~]# cat nsdlist[size=7.5pt]sdd:node1:node2:dataAndMetadata::[size=7.5pt][root@node1 ~]#[size=7.5pt][root@node1 ~]# mmcrnsd -F nsdlist[size=7.5pt]mmcrnsd: Processingdisk sdd[size=7.5pt]mmcrnsd:Propagating the cluster configuration data to all[size=7.5pt] affected nodes. This is an asynchronous process.[size=7.5pt][root@node1 ~]#[size=7.5pt][root@node1 ~]#mmlsnsd[size=7.5pt] [size=7.5pt] File system Disk name NSD servers[size=7.5pt]---------------------------------------------------------------------------[size=7.5pt] (free disk) gpfs1nsd node1,node2[size=7.5pt] [size=7.5pt][root@node1 ~]#[size=7.5pt][root@node1 ~]# mmstartup -a[size=7.5pt]Sun Nov 24 17:12:20PST 2013: mmstartup: Starting GPFS ...[size=7.5pt][root@node1 ~]#[size=7.5pt][root@node1 ~]# mmcrfs gpfs1nsd -F nsdlist[size=7.5pt] [size=7.5pt]The following disksof gpfs1nsd will be formatted on node node1:[size=7.5pt] gpfs1nsd: size 41943040 KB[size=7.5pt]Formatting filesystem ...[size=7.5pt]Disks up to size419 GB can be added to storage pool 'system'.[size=7.5pt]Creating Inode File[size=7.5pt]Creating AllocationMaps[size=7.5pt]Clearing InodeAllocation Map[size=7.5pt]Clearing BlockAllocation Map[size=7.5pt]FormattingAllocation Map for storage pool 'system'[size=7.5pt]Completed creationof file system /dev/gpfs1nsd.[size=7.5pt]mmcrfs: Propagatingthe cluster configuration data to all[size=7.5pt] affected nodes. This is an asynchronous process.[size=7.5pt][root@node1 ~]#[size=7.5pt][root@node1 ~]#mmlsnsd[size=7.5pt] [size=7.5pt] File system Disk name NSD servers[size=7.5pt]---------------------------------------------------------------------------[size=7.5pt] gpfs1nsd gpfs1nsd node1,node2[size=7.5pt] [size=7.5pt][root@node1 ~]#[size=7.5pt][root@node1~]# ls -l /dev/gpfs1nsd[size=7.5pt]brw-r--r-- 1 rootroot 239, 150 Nov 24 17:12 /dev/gpfs1nsd[size=7.5pt][root@node1 ~]# sshnode2[size=7.5pt]Last login: Mon Nov25 11:34:15 2013 from 192.168.71.1[size=7.5pt][root@node2 ~]# ls -l /dev/gpfs1nsd[size=7.5pt]brw-r--r-- 1 rootroot 239, 150 Nov 25 11:43 /dev/gpfs1nsd[size=7.5pt][root@node2 ~]#[size=7.5pt][root@node1~]# cat /etc/fstab[size=7.5pt]LABEL=/ / ext3 defaults 1 1[size=7.5pt]LABEL=/boot /boot ext3 defaults 1 2[size=7.5pt]tmpfs /dev/shm tmpfs defaults 0 0[size=7.5pt]devpts /dev/pts devpts gid=5,mode=620 0 0[size=7.5pt]sysfs /sys sysfs defaults 0 0[size=7.5pt]proc /proc proc defaults 0 0[size=7.5pt]LABEL=SWAP-sda3 swap swap defaults 0 0[size=7.5pt]/dev/gpfs1nsd /gpfs/gpfs1nsd gpfs rw,mtime,atime,dev=gpfs1nsd,autostart 00[size=7.5pt] [size=7.5pt] [size=7.5pt][root@node2 ~]# cat /etc/fstab[size=7.5pt]LABEL=/ / ext3 defaults 1 1[size=7.5pt]LABEL=/boot /boot ext3 defaults 1 2[size=7.5pt]tmpfs /dev/shm tmpfs defaults 0 0[size=7.5pt]devpts /dev/pts devpts gid=5,mode=620 0 0[size=7.5pt]sysfs /sys sysfs defaults 0 0[size=7.5pt]proc /proc proc defaults 0 0[size=7.5pt]LABEL=SWAP-sda3 swap swap defaults 0 0[size=7.5pt]/dev/gpfs1nsd /gpfs/gpfs1nsd gpfs rw,mtime,atime,dev=gpfs1nsd,autostart 00[size=7.5pt] [size=7.5pt][root@node2 ~]#mount /gpfs/gpfs1nsd[size=7.5pt][root@node2 ~]# df-h[size=7.5pt]Filesystem Size Used Avail Use% Mounted on[size=7.5pt]/dev/sda2 19G 2.8G 15G 16% /[size=7.5pt]/dev/sda1 289M 16M 259M 6% /boot[size=7.5pt]tmpfs 252M 0 252M 0% /dev/shm[size=7.5pt]/dev/gpfs1nsd 40G 165M 40G 1% /gpfs/gpfs1nsd[size=7.5pt][root@node2 ~]#[size=7.5pt] [size=7.5pt] [size=7.5pt]