1.11. Isambard 2

The requirements for the machine learning project on the blade servers:

Current purpose - re-run just the steady state case without optimisation. Only require:

1.11.1. Proposed Run Script (no run functions - check if everything is installed)

module load python/anaconda3
source activate py3
of4x

pyFoamClearCase.py . --processors-remove
/usr/local/Pointwise/PointwiseV18.0R2/pointwise -b ./Hydro_V18_wider.glf
rm -r 0
cp -r 0_orig 0
transformPoints -scale '(0.0254 0.0254 0.0254)'
renumberMesh -overwrite
cp system/controlDict_simple system/controlDict
decomposePar
mpirun -np 28 potentialFoam -parallel > log.potentialFoam 2>&1
mpirun -np 28 simpleFoam -parallel > log.simpleFoam 2>&1
reconstructPar -latestTime
pyFoamCopyLastToFirst.py . .
pyFoamClearCase.py . --processors-remove
cp system/controlDict_parcel system/controlDict
decomposePar
mpirun -np 28 icoUncoupledKinematicParcelFoam -parallel > log.icoUncoupledKinematicParcelFoam 2>&1

With run functions:

module load python/anaconda3
source activate py3
module load mpi/openmpi-x86_64
source /usr/local/OpenFOAM/OpenFOAM-4.x/etc/bashrc
export PS1="(OF:\$WM_PROJECT_VERSION) $PS1"

. $WM_PROJECT_DIR/bin/tools/RunFunctions
echo "Environment setup complete"
rm -rf log.*
pyFoamClearCase.py . --processors-remove
echo "Case cleared"
renumberMesh -overwrite
echo "Renumbered mesh"
cp system/controlDict_simple system/controlDict
echo "Copied controlDict"
decomposePar
echo "Decomposed case"
runParallel potentialFoam
echo "PotentialFoam finished"
runParallel simpleFoam
echo "SimpleFoam finished"
reconstructPar -latestTime
echo "Case reconstructed"
pyFoamCopyLastToFirst.py . .
echo "Copied first to last"
pyFoamClearCase.py . --processors-remove
echo "Cleared case"
cp system/controlDict_parcel system/controlDict
echo "Copied controlDict"
decomposePar
echo "Decomposed case"
runParallel icoUncoupledKinematicParcelFoam
echo "IcoUncoupledKinematicParcelFoam finished"

1.11.2. OS - Phase 1 Isambard

uname -a
Linux login-01 3.10.0-514.el7.x86_64 #1 SMP Wed Oct 19 11:24:13 EDT 2016 x86_64 x86_64 x86_64 GNU/Linux

1.11.3. Anaconda 3.5.4

    ssh power-001
source /opt/anaconda3/latest/bin/activate

1.11.4. Python 3.x

  • Change to python 3 environment

conda search "python"
conda create -n py3 python=3.5.4 anaconda

Proceed ([y]/n)?

y

source activate py3

environment location: /home/ex-aroberts/.conda/envs/py3

1.11.4.1. pyFoam 0.6.6

pip install PyFoam==0.6.6
Collecting PyFoam==0.6.6
Cache entry deserialization failed, entry ignored
Cache entry deserialization failed, entry ignored
Downloading https://files.pythonhosted.org/packages/aa/be/47eb7b9b0533af692a494d0bb3a6820f00d01f3b7ee8d3743b07475a5885/PyFoam-0.6.6-py2.py3-none-any.whl (608kB)
100% |████████████████████████████████| 614kB 609kB/s
Requirement already satisfied: numpy in ./.conda/envs/py3/lib/python3.5/site-packages (from PyFoam==0.6.6)
Installing collected packages: PyFoam
Successfully installed PyFoam-0.6.6
You are using pip version 9.0.1, however version 19.2.2 is available.
You should consider upgrading via the 'pip install --upgrade pip' command.


    pyFoamVersion.py
    Machine info: Linux  power-001.gw4.metoffice.gov.uk  4.14.0-49.13.1.el7a.ppc64le #1 SMP Mon Aug 27 07:37:11 EDT 2018  ppc64le  ppc64le

    Python version: 3.5.4 Anaconda, Inc. (default, Feb  2 2018, 12:32:07)
    [GCC 7.2.0]

    Python executable: /home/ex-aroberts/.conda/envs/py3/bin/python

    Python 3 is supported with PyFoam
    PYTHONPATH is not set

    Location of this utility: /home/ex-aroberts/.conda/envs/py3/bin/pyFoamVersion.py

    PyFoam WARNING on line 201 of file /home/ex-aroberts/.conda/envs/py3/lib/python3.5/site-packages/PyFoam/FoamInformation.py : Basedir /home/ex-aroberts/foam for fork extend does not exist or is not a directory
    Version () (reported as number  )Fork openfoam of the installed 0 versions

    pyFoam-Version: 0.6.6

1.11.4.2. Numpy 1.13.3

pip install numpy

Requirement already satisfied: numpy in ./.conda/envs/py3/lib/python3.5/site-packages
You are using pip version 9.0.1, however version 19.2.2 is available.
You should consider upgrading via the 'pip install --upgrade pip' command.

In [1]: import numpy

In [2]: print (numpy.__version__)
1.13.3

1.11.4.3. Matplotlib 2.1.2

pip install matplotlib

Requirement already satisfied: matplotlib in ./.conda/envs/py3/lib/python3.5/site-packages
Requirement already satisfied: numpy>=1.7.1 in ./.conda/envs/py3/lib/python3.5/site-packages (from matplotlib)
Requirement already satisfied: six>=1.10 in ./.conda/envs/py3/lib/python3.5/site-packages (from matplotlib)
Requirement already satisfied: python-dateutil>=2.1 in ./.conda/envs/py3/lib/python3.5/site-packages (from matplotlib)
Requirement already satisfied: pytz in ./.conda/envs/py3/lib/python3.5/site-packages (from matplotlib)
Requirement already satisfied: cycler>=0.10 in ./.conda/envs/py3/lib/python3.5/site-packages (from matplotlib)
Requirement already satisfied: pyparsing!=2.0.4,!=2.1.2,!=2.1.6,>=2.0.1 in ./.conda/envs/py3/lib/python3.5/site-packages (from matplotlib)
You are using pip version 9.0.1, however version 19.2.2 is available.
You should consider upgrading via the 'pip install --upgrade pip' command.

In [3]: import matplotlib

In [4]: print (matplotlib.__version__) 2.1.2

In [5]:

1.11.4.4. scipy

pip install scipy

Requirement already satisfied: scipy in ./.conda/envs/py3/lib/python3.5/site-packages You are using pip version 9.0.1, however version 19.2.2 is available. You should consider upgrading via the ‘pip install –upgrade pip’ command.

In [5]: import scipy

In [6]: print (scipy.__version__) 1.0.0

In [7]:

1.11.4.5. deap 1.3

pip install deap

Collecting deap Cache entry deserialization failed, entry ignored Cache entry deserialization failed, entry ignored Downloading https://files.pythonhosted.org/packages/f7/d2/ca7c4ebbd2c30a017c188b72da98dab3ab54648c3b0aaa7a2efa772d6147/deap-1.3.0.tar.gz (1.1MB) 100% 1.1MB 495kB/s Building wheels for collected packages: deap Running setup.py bdist_wheel for deap … done Stored in directory: /home/ex-aroberts/.cache/pip/wheels/95/46/10/82f914655ecf6ff1dc7d42a47ba704a9c9181f24f8edfbe75d Successfully built deap Installing collected packages: deap Successfully installed deap-1.3.0 You are using pip version 9.0.1, however version 19.2.2 is available. You should consider upgrading via the ‘pip install –upgrade pip’ command.

In [7]: import deap

In [8]: print (deap.__version__) 1.3

1.11.4.6. PyDOE 0.3.8

pip install pyDOE

Collecting pyDOE Cache entry deserialization failed, entry ignored Cache entry deserialization failed, entry ignored Downloading https://files.pythonhosted.org/packages/bc/ac/91fe4c039e2744466621343d3b8af4a485193ed0aab53af5b1db03be0989/pyDOE-0.3.8.zip Requirement already satisfied: numpy in ./.conda/envs/py3/lib/python3.5/site-packages (from pyDOE) Requirement already satisfied: scipy in ./.conda/envs/py3/lib/python3.5/site-packages (from pyDOE) Building wheels for collected packages: pyDOE Running setup.py bdist_wheel for pyDOE … done Stored in directory: /home/ex-aroberts/.cache/pip/wheels/7c/c8/58/a6493bd415e8ba5735082b5e0c096d7c1f2933077a8ce34544 Successfully built pyDOE Installing collected packages: pyDOE Successfully installed pyDOE-0.3.8 You are using pip version 9.0.1, however version 19.2.2 is available. You should consider upgrading via the ‘pip install –upgrade pip’ command.

In [10]: import pyDOE

In [11]: print (pyDOE.__version__) 0.3.8

1.11.4.7. evoalgos

pip install evoalgos

Collecting evoalgos Cache entry deserialization failed, entry ignored Cache entry deserialization failed, entry ignored Downloading https://files.pythonhosted.org/packages/89/f0/28f113784fb34485b801a8dad0c16ac84220270f3451e400d0964c5df02b/evoalgos-1.0.tar.gz (41kB) 100% 51kB 573kB/s Collecting optproblems (from evoalgos) Cache entry deserialization failed, entry ignored Cache entry deserialization failed, entry ignored Downloading https://files.pythonhosted.org/packages/3e/4d/9e23369c07e75922a1d472270348fc907581fc4bb3fde9d132e9f61def49/optproblems-1.2.tar.gz (2.0MB) 100% 2.0MB 398kB/s Requirement already satisfied: numpy in ./.conda/envs/py3/lib/python3.5/site-packages (from evoalgos) Building wheels for collected packages: evoalgos, optproblems Running setup.py bdist_wheel for evoalgos … done Stored in directory: /home/ex-aroberts/.cache/pip/wheels/e7/6c/66/3a4dabe727f440c69f5ca2e62322f33718a7bf58f78641221e Running setup.py bdist_wheel for optproblems … done Stored in directory: /home/ex-aroberts/.cache/pip/wheels/76/a8/de/c28b18f67d9e945888b6c4f4b5f76037f5e1b11ab9971d1af2 Successfully built evoalgos optproblems Installing collected packages: optproblems, evoalgos Successfully installed evoalgos-1.0 optproblems-1.2 You are using pip version 9.0.1, however version 19.2.2 is available. You should consider upgrading via the ‘pip install –upgrade pip’ command.

In [12]: import evoalgos

In [13]: print (evoalgos.__version__) 1.0

1.11.4.8. GPy

pip install GPy

Collecting GPy Cache entry deserialization failed, entry ignored Cache entry deserialization failed, entry ignored Downloading https://files.pythonhosted.org/packages/5c/a1/4418ee1db50c6e917e399db841c30b95d3c242555272e85473404ab06377/GPy-1.9.8.tar.gz (989kB) 100% 993kB 534kB/s Requirement already satisfied: numpy>=1.7 in ./.conda/envs/py3/lib/python3.5/site-packages (from GPy) Requirement already satisfied: scipy>=0.16 in ./.conda/envs/py3/lib/python3.5/site-packages (from GPy) Requirement already satisfied: six in ./.conda/envs/py3/lib/python3.5/site-packages (from GPy) Collecting paramz>=0.9.0 (from GPy) Cache entry deserialization failed, entry ignored Cache entry deserialization failed, entry ignored Downloading https://files.pythonhosted.org/packages/d8/37/4abbeb78d30f20d3402887f46e6e9f3ef32034a9dea65d243654c82c8553/paramz-0.9.5.tar.gz (71kB) 100% 71kB 1.8MB/s Requirement already satisfied: decorator>=4.0.10 in ./.conda/envs/py3/lib/python3.5/site-packages (from paramz>=0.9.0->GPy) Building wheels for collected packages: GPy, paramz Running setup.py bdist_wheel for GPy … done Stored in directory: /home/ex-aroberts/.cache/pip/wheels/96/ee/cd/1c4dd7df63246b1e8de58af6d4457b7aed13509fdc0c918a13 Running setup.py bdist_wheel for paramz … done Stored in directory: /home/ex-aroberts/.cache/pip/wheels/c8/4a/0e/6e0dc85541825f991c431619e25b870d4b812c911214690cf8 Successfully built GPy paramz Installing collected packages: paramz, GPy Successfully installed GPy-1.9.8 paramz-0.9.5 You are using pip version 9.0.1, however version 19.2.2 is available. You should consider upgrading via the ‘pip install –upgrade pip’ command.

In [14]: import GPy

In [15]: print (GPy.__version__) 1.9.8

1.11.4.9. CMA

pip install cma

Collecting cma Cache entry deserialization failed, entry ignored Cache entry deserialization failed, entry ignored Downloading https://files.pythonhosted.org/packages/b9/3b/87a4efbcfeaf3172d81ef843f0b0c34c3ba60ec884aa6777f34f68b57418/cma-2.7.0-py2.py3-none-any.whl (239kB) 100% 245kB 669kB/s Installing collected packages: cma Successfully installed cma-2.7.0 You are using pip version 9.0.1, however version 19.2.2 is available. You should consider upgrading via the ‘pip install –upgrade pip’ command.

In [16]: import cma

In [17]: print (cma.__version__) 2.7.0 $Revision: 4426 $ $Date: 2019-04-24 18:03:09 +0200 (Wed, 24 Apr 2019) $

In [18]:

1.11.5. Python on Phase 2

1.11.5.1. environment

module load cray-python/3.6.5.6
virtualenv --system-site-packages --python=python3.6 myPy
    Running virtualenv with interpreter /opt/python/3.6.5.6/bin/python3.6
    Using base prefix '/opt/python/3.6.5.6'
    New python executable in /lustre/home/ex-aroberts/py3/bin/python3.6
    Also creating executable in /lustre/home/ex-aroberts/py3/bin/python
    Installing setuptools, pip, wheel...done.

source ~/myPy/bin/activate

1.11.5.2. pyfoam 0.6.6

pip install –user PyFoam==0.6.6

export PATH=$HOME/.local/bin:$PATH

export PYTHONPATH=$HOME/.local/lib/python3.6/site-packages:$PYTHONPATH

1.11.5.3. Numpy 1.15.1

pip install --user numpy

In [1]: import numpy

In [2]: print (numpy.__version__)
1.15.1

1.11.5.4. Matplotlib 3.1.1

pip install --user matplotlib

In [3]: import matplotlib

In [4]: print (matplotlib.__version__)
3.1.1

1.11.5.5. scipy 1.1.0

pip install --user scipy

In [5]: import scipy

In [6]: print (scipy.__version__)
1.1.0

1.11.5.6. deap 1.3

pip install --user deap

In [7]: import deap

In [8]: print (deap.__version__)
1.3

1.11.5.7. PyDOE 0.3.8

pip install --user pyDOE

In [10]: import pyDOE

In [11]: print (pyDOE.__version__)
0.3.8

1.11.5.8. evoalgos

pip install --user evoalgos

In [12]: import evoalgos

In [13]: print (evoalgos.__version__)
1.0

1.11.5.9. GPy 1.9.8

On Isambard, the Gaussian Process module fails to import stationary, coregionalize and choleskies cython modules and uses numpy instead. Is this a problem for efficiency?

pip install --user GPy

    python
In [14]: import GPy

In [15]: print (GPy.__version__)
1.9.8

warning in stationary: failed to import cython module: falling back to numpy
warning in coregionalize: failed to import cython module: falling back to numpy
warning in choleskies: failed to import cython module: falling back to numpy

1.11.5.10. CMA 2.7.0

pip install --user cma

In [16]: import cma

In [17]: print (cma.__version__)
2.7.0  $Revision: 4426 $ $Date: 2019-04-24 18:03:09 +0200 (Wed, 24 Apr 2019) $

1.11.5.11. numpy-stl

pip install --user numpy-stl

In [16]: from stl import mesh

1.11.5.12. gecco-2017

gecco-2017 code works on Isambard, but GPy uses numpy not cython for three modules.

git clone https://bitbucket.org/arahat/gecco-2017/src/master/

    cd FrontCalc
    python setup.py install --user

Output:

running install
running build
running build_ext
running install_lib
copying build/lib.linux-aarch64-3.6/_csupport.cpython-36m-aarch64-linux-gnu.so -> /home/ex-aroberts/.local/lib/python3.6/site-packages
running install_egg_info
Writing /home/ex-aroberts/.local/lib/python3.6/site-packages/UNKNOWN-0.0.0-py3.6.egg-info

from IPython.display import clear_output

import numpy as np
import IscaOpt
warning in stationary: failed to import cython module: falling back to numpy
warning in coregionalize: failed to import cython module: falling back to numpy
warning in choleskies: failed to import cython module: falling back to numpy

settings = {\
        'n_dim': 6,\
        'n_obj': 2,\
        'lb': np.zeros(6),\
        'ub': np.ones(6),\
        'ref_vector': [2.5]*2,\
        'method_name': 'HypI',\
        'budget':100,\
        'n_samples':65,\
        'visualise':True,\
        'multisurrogate':False}

# function settings
from deap import benchmarks as BM
fun = BM.dtlz2
args = (2,) # number of oag stationarybjectives as argument

# optimise
res = IscaOpt.Optimiser.EMO(fun, args, settings=settings)
clear_output()

1.11.6. OpenFOAM 4.x on Phase 1 (doesn’t work out of the box)

ssh power-001

Download packages:

git clone https://github.com/OpenFOAM/OpenFOAM-4.x.git

git clone https://github.com/OpenFOAM/ThirdParty-4.x.git

Don’t need to install any software for compilation

Set environment variables:

cd $𝐻𝑂𝑀𝐸

mpirun –version

mpirun (Open MPI) 1.10.7

gcc –version

gcc (GCC) 4.8.5 20150623 (Red Hat 4.8.5-28)

cmake –version

cmake version 2.8.12.2

alias of4x=’module load mpi/openmpi-ppc64le; source $HOME/OpenFOAM/OpenFOAM-4.x/etc/bashrc; export PS1=”(LOCAL POWER OF:$WM_PROJECT_VERSION) $PS1”’

After logging out and logging in again, install Third Party Scotch and PT Scotch

of4x

foamSystemCheck

cd $WM_THIRD_PARTY_DIR

./Allwmake > log.make 2>&1 &

Install OpenFOAM

cd ../OpenFOAM-4.x/

./Allwmake -j 16 > log.make 2>&1 &

Compiling enabled on 16 cores gcc -m64 -mcpu=power8 -DlinuxPPC64le -DWM_ARCH_OPTION=64 -DWM_DP -DWM_LABEL_SIZE=32 -Wall -O3 -fPIC dirToString.c -o /lustre-nfs/home/ex-aroberts/OpenFOAM/OpenFOAM-4.x/wmake/platforms/linuxPPC64leGcc/dirToString flex wmkdep.l make: flex: Command not found make: [/lustre-nfs/home/ex-aroberts/OpenFOAM/OpenFOAM-4.x/wmake/platforms/linuxPPC64leGcc/wmkdep] Error 127

1.11.6.1. 1812

2019-08-21 21:16:15 +0000

Finished compile of cfMesh with OpenFOAM-v1812 Gcc system compiler linux64GccDPInt32Opt, with SYSTEMOPENMPI openmpi-system

2019-08-21 21:16:15 +0000

OpenFOAM-v1812 Gcc system compiler linux64GccDPInt32Opt, with SYSTEMOPENMPI openmpi-system

api = 1812 patch = 0 bin = 277 entries lib = 122 entries

Done logging to ‘log.linux64GccDPInt32Opt’

1.11.7. Output of PATH

(APR OF:4.x) (py3) $ echo $PATH

/misc/OpenFOAM/ThirdParty-4.x/platforms/linux64Gcc/gperftools-svn/bin:
/misc/OpenFOAM/ThirdParty-4.x/platforms/linux64Gcc/ParaView-5.0.1/bin:
/home/links/apr207/OpenFOAM/apr207-4.x/platforms/linux64GccDPInt32Opt/bin:
/misc/OpenFOAM/site/4.x/platforms/linux64GccDPInt32Opt/bin:
/misc/OpenFOAM/OpenFOAM-4.x/platforms/linux64GccDPInt32Opt/bin:
/misc/OpenFOAM/OpenFOAM-4.x/bin:
/misc/OpenFOAM/OpenFOAM-4.x/wmake:
/usr/lib64/openmpi/bin:
/home/links/apr207/.conda/envs/py3/bin:
/usr/local/anaconda3/bin:
/usr/local/texlive/2017/bin/x86_64-linux:
/usr/lib64/qt-3.3/bin:
/usr/local/bin:
/usr/bin:
/usr/local/sbin:
/usr/sbin:
/opt/puppetlabs/bin:
/home/links/apr207/.local/bin:
/home/links/apr207/bin

1.11.8. Pointwise 18.0R2

sh pw-V18.0R2-linux_x86_64-jre.sh
Next
Accept
/home/links/apr207/Pointwise/PointwiseV18.0R2 is installation directory
Run Pointwise
(Won't find license)
Specify license server
server: emps-pointwise
port: 2385

Add this to .bashrc:

# Pointwise alias for this version
alias pointwise="/home/links/apr207/Pointwise/PointwiseV18.0R2/pointwise"

1.11.9. Local Install of OpenFOAM on Isambard (Phase 2)

  1. Download OpenFOAM-v1812 and ThirdParty-v1812

BUILD_DIR=$HOME/OpenFOAM
mkdir $BUILD_DIR
cd $BUILD_DIR
wget https://sourceforge.net/projects/openfoamplus/files/v1812/OpenFOAM-v1812.tgz
wget https://sourceforge.net/projects/openfoamplus/files/v1812/ThirdParty-v1812.tgz
tar -xvf OpenFOAM-v1812.tgz -C $BUILD_DIR
tar -xvf ThirdParty-v1812.tgz -C $BUILD_DIR

  1. Load ARM compilers

# load ARM compiler if no compilers are loaded
module load PrgEnv-allinea/6.0.5
# or swap current cray compiler with arm compiler
module switch PrgEnv-cray/6.0.5 PrgEnv-allinea/6.0.5

  1. Make modifications to the configuration files required to build OF

# Download the two patch files and apply the patch running the below commands
wget https://dl.dropboxusercontent.com/s/jnpjop7kll5f3ph/OpenFOAM-v1812.patch
wget https://dl.dropboxusercontent.com/s/q0iwu2qf7pv9s6n/ThirdParty-v1812.patch
(cd OpenFOAM-v1812/; patch -p1 < $BUILD_DIR/OpenFOAM-v1812.patch)
(cd ThirdParty-v1812/; patch -p1 < $BUILD_DIR/ThirdParty-v1812.patch)
# create prefs.sh file with compiler options
cd $BUILD_DIR/OpenFOAM-v1812
echo -e "WM_COMPILER=Arm \nWM_MPLIB=CRAY-MPICH \nWM_LABEL_SIZE=64" > etc/prefs.sh
# add the following to $HOME/.bashrc:
module swap PrgEnv-cray PrgEnv-allinea
alias localof1812='module load PrgEnv-allinea; export OPENFOAM_DIR=$HOME/OpenFOAM/OpenFOAM-v1812; export PATH=$PATH:$OPENFOAM_DIR/bin/; source $OPENFOAM_DIR/etc/bashrc; export PS1="(LOCAL OF:\$WM_PROJECT_VERSION) $PS1"'
# reload bashrc (exit and relogin)

  1. Compile OpenFOAM-v1812

# This step is optional to check if everything is okay to build OpenFOAM
foamSystemCheck
# Compile openfoam using all available processors (-j) with reduced output (-s) and log the output (-l) to a file so that we can examine any compilation issues later.
./Allwmake -j -s -l

  1. Example run script

#!/bin/bash
#PBS -N openfoam_job
# Select 1 nodes (maximum of 64 cores)
#PBS -l select=1:ncpus=4
# Select wall time to 1 hour
#PBS -l walltime=1:00:00
# Use the arm nodes
#PBS -q arm
# Load modules for currently recommended OpenFOAM v1812 build
module load PrgEnv-allinea
# Change to directory that script was submitted from
export PBS_O_WORKDIR=$(readlink -f $PBS_O_WORKDIR)
export OMP_NUM_THREADS=1
cd $PBS_O_WORKDIR
# Load the environment variables for OpenFOAM v1812 build
export OPENFOAM_DIR=$HOME/OpenFOAM/OpenFOAM-v1812
export PATH=$PATH:$OPENFOAM_DIR/bin/
source $OPENFOAM_DIR/etc/bashrc
# Clear folder
rm -rf log
rm -rf log.*
rm -rf openfoam*
rm -rf processor*
rm -rf 0.*
rm -rf postProcessing
# Launch OpenFoam solvers
blockMesh > log.blockMesh 2>&1
checkMesh > log.checkMesh 2>&1
decomposePar > log.decomposePar 2>&1
aprun -n 4 icoFoam -parallel > log.icoFoam 2>&1
reconstructPar -time 0: > log.reconstructPar 2>&1