1.7. Isambard

The requirements for the machine learning project on the blade servers:

Current purpose - re-run just the steady state case without optimisation. Only require:

1.7.1. Proposed Run Script (no run functions - check if everything is installed)

module load python/anaconda3
source activate py3
of4x

pyFoamClearCase.py . --processors-remove
/usr/local/Pointwise/PointwiseV18.0R2/pointwise -b ./Hydro_V18_wider.glf
rm -r 0
cp -r 0_orig 0
transformPoints -scale '(0.0254 0.0254 0.0254)'
renumberMesh -overwrite
cp system/controlDict_simple system/controlDict
decomposePar
aprun -np 28 potentialFoam -parallel > log.potentialFoam 2>&1
aprun -np 28 simpleFoam -parallel > log.simpleFoam 2>&1
reconstructPar -latestTime
pyFoamCopyLastToFirst.py . .
pyFoamClearCase.py . --processors-remove
cp system/controlDict_parcel system/controlDict
decomposePar
aprun -np 28 icoUncoupledKinematicParcelFoam -parallel > log.icoUncoupledKinematicParcelFoam 2>&1

1.7.2. OS

uname -a
Linux xcil00 4.12.14-25.22_5.0.70-cray_ari_s #1 SMP Tue Mar 5 00:08:48 UTC 2019 (bc16c54) aarch64 aarch64 aarch64 GNU/Linux

1.7.3. Anaconda 3.5.4

https://anaconda.org/anaconda/python/files?version=3.5.4

1.7.4. Python 3.x

  • Change to python 3 environment

conda search "python"
conda create -n py3 python=3.5.4 anaconda

Proceed ([y]/n)?

y
source activate py3

1.7.4.1. pyFoam 0.6.6

pip install PyFoam==0.6.6

1.7.4.2. Numpy

pip install numpy

1.7.4.3. Matplotlib

pip install matplotlib

1.7.4.4. scipy

pip install scipy

1.7.4.5. deap

pip install deap

1.7.4.6. PyDOE

pip install pyDOE

1.7.4.7. evoalgos

pip install evoalgos

1.7.4.8. GPy

pip install GPy

1.7.4.9. CMA

pip install cma

1.7.5. OpenFOAM 4.x

of4
foamVersion
OpenFOAM-4.x

1.7.6. Output of PATH

(APR OF:4.x) (py3) $ echo $PATH
/misc/OpenFOAM/ThirdParty-4.x/platforms/linux64Gcc/gperftools-svn/bin:
/misc/OpenFOAM/ThirdParty-4.x/platforms/linux64Gcc/ParaView-5.0.1/bin:
/home/links/apr207/OpenFOAM/apr207-4.x/platforms/linux64GccDPInt32Opt/bin:
/misc/OpenFOAM/site/4.x/platforms/linux64GccDPInt32Opt/bin:
/misc/OpenFOAM/OpenFOAM-4.x/platforms/linux64GccDPInt32Opt/bin:
/misc/OpenFOAM/OpenFOAM-4.x/bin:
/misc/OpenFOAM/OpenFOAM-4.x/wmake:
/usr/lib64/openmpi/bin:
/home/links/apr207/.conda/envs/py3/bin:
/usr/local/anaconda3/bin:
/usr/local/texlive/2017/bin/x86_64-linux:
/usr/lib64/qt-3.3/bin:
/usr/local/bin:
/usr/bin:
/usr/local/sbin:
/usr/sbin:
/opt/puppetlabs/bin:
/home/links/apr207/.local/bin:
/home/links/apr207/bin

1.7.7. Pointwise 18.0R2

sh pw-V18.0R2-linux_x86_64-jre.sh
Next
Accept
/home/links/apr207/Pointwise/PointwiseV18.0R2 is installation directory
Run Pointwise
(Won't find license)
Specify license server
server: emps-pointwise
port: 2385

Add this to .bashrc:

# Pointwise alias for this version
alias pointwise="/home/links/apr207/Pointwise/PointwiseV18.0R2/pointwise"

1.7.8. Local Install of OpenFOAM on Isambard

  1. Download OpenFOAM-v1812 and ThirdParty-v1812

BUILD_DIR=$HOME/OpenFOAM
mkdir $BUILD_DIR
cd $BUILD_DIR
wget https://sourceforge.net/projects/openfoamplus/files/v1812/OpenFOAM-v1812.tgz
wget https://sourceforge.net/projects/openfoamplus/files/v1812/ThirdParty-v1812.tgz
tar -xvf OpenFOAM-v1812.tgz -C $BUILD_DIR
tar -xvf ThirdParty-v1812.tgz -C $BUILD_DIR
  1. Load ARM compilers

# load ARM compiler if no compilers are loaded
module load PrgEnv-allinea/6.0.5
# or swap current cray compiler with arm compiler
module switch PrgEnv-cray/6.0.5 PrgEnv-allinea/6.0.5
  1. Make modifications to the configuration files required to build OF

# Download the two patch files and apply the patch running the below commands
wget https://dl.dropboxusercontent.com/s/jnpjop7kll5f3ph/OpenFOAM-v1812.patch
wget https://dl.dropboxusercontent.com/s/q0iwu2qf7pv9s6n/ThirdParty-v1812.patch
(cd OpenFOAM-v1812/; patch -p1 < $BUILD_DIR/OpenFOAM-v1812.patch)
(cd ThirdParty-v1812/; patch -p1 < $BUILD_DIR/ThirdParty-v1812.patch)
# create prefs.sh file with compiler options
cd $BUILD_DIR/OpenFOAM-v1812
echo -e "WM_COMPILER=Arm \nWM_MPLIB=CRAY-MPICH \nWM_LABEL_SIZE=64" > etc/prefs.sh
# add the following to $HOME/.bashrc:
module swap PrgEnv-cray PrgEnv-allinea
alias localof1812='module load PrgEnv-allinea; export OPENFOAM_DIR=$HOME/OpenFOAM/OpenFOAM-v1812; export PATH=$PATH:$OPENFOAM_DIR/bin/; source $OPENFOAM_DIR/etc/bashrc; export PS1="(LOCAL OF:\$WM_PROJECT_VERSION) $PS1"'
# reload bashrc (exit and relogin)
  1. Compile OpenFOAM-v1812

# This step is optional to check if everything is okay to build OpenFOAM
foamSystemCheck
# Compile openfoam using all available processors (-j) with reduced output (-s) and log the output (-l) to a file so that we can examine any compilation issues later.
./Allwmake -j -s -l
  1. Example run script

#!/bin/bash
#PBS -N openfoam_job
# Select 1 nodes (maximum of 64 cores)
#PBS -l select=1:ncpus=4
# Select wall time to 1 hour
#PBS -l walltime=1:00:00
# Use the arm nodes
#PBS -q arm
# Load modules for currently recommended OpenFOAM v1812 build
module load PrgEnv-allinea
# Change to directory that script was submitted from
export PBS_O_WORKDIR=$(readlink -f $PBS_O_WORKDIR)
export OMP_NUM_THREADS=1
cd $PBS_O_WORKDIR
# Load the environment variables for OpenFOAM v1812 build
export OPENFOAM_DIR=$HOME/OpenFOAM/OpenFOAM-v1812
export PATH=$PATH:$OPENFOAM_DIR/bin/
source $OPENFOAM_DIR/etc/bashrc
# Clear folder
rm -rf log
rm -rf log.*
rm -rf openfoam*
rm -rf processor*
rm -rf 0.*
rm -rf postProcessing
# Launch OpenFoam solvers
blockMesh > log.blockMesh 2>&1
checkMesh > log.checkMesh 2>&1
decomposePar > log.decomposePar 2>&1
aprun -n 4 icoFoam -parallel > log.icoFoam 2>&1
reconstructPar -time 0: > log.reconstructPar 2>&1