Differences

This shows you the differences between two versions of the page.

Link to this comparison view

Both sides previous revision Previous revision
doku:copy [2021/05/13 17:48]
goldenberg removed
— (current)
Line 1: Line 1:
-<code> 
  
-################################################ 
-#####         start copying process         #### 
-################################################ 
- 
- 
- 
-#temporary directory on nodes: 
-tmp_dir=$TMPDIR/data 
- 
-#naming of the tar files that should be distributed to 
-#the nodes. Each tar file should contain 8 subdirs. 
- 
-#file names are completed by appending a number without leading 
-#zeros + tar.gz 
-input_tar_file_base=processes_ 
- 
-#naming of the outputfiles; number and tar.gz are appended automatically 
-output_tar_base=output_ 
-#define which directories and files should be put into the output tar.gz 
-# not tested yet 
-PACKING=data\* 
- 
-start_dir=`pwd` 
-#save output data to this directory 
-data_save="${start_dir}/data_${JOB_ID}" 
- 
- 
- 
-nodes_uniq=$(cat $TMPDIR/machines| uniq) 
- 
- 
-#copy files per node 
-#tar.gz files contain 8 subdirectories for each process 
- 
-j=0 
- 
-for i in $nodes_uniq 
-do 
-    tar_file_name="${input_tar_file_base}${j}.tar.gz" 
-    echo "creating ${tmp_dir} on $i" 
-    ssh $i mkdir -p $tmp_dir 
-    echo "copying $tar_file_name to node $i" 
-    ssh $i cp ${start_dir}\/${tar_file_name} ${tmp_dir} 
-    ssh $i cp -r ${start_dir}\/tmp_dictionaries\/* ${tmp_dir} 
-    echo "extracting file" 
-    ssh $i "cd ${tmp_dir} ;tar -zxf ${tmp_dir}\/${tar_file_name}" 
-    j=$(echo "$j+1"|bc -l) 
-done 
- 
-#command to run: 
-cd ${tmp_dir} 
-time mpirun -mca btl_openib_ib_timeout 20 -machinefile $TMPDIR/machines -np 64 $1 -parallel 
- 
-#cp files per node back to start directory of job 
-echo "=================================================" 
-j=0 
-for i in $nodes_uniq 
-do 
-    tar_file_name="${input_tar_file_base}${j}.tar.gz" 
-    output_tar_file="${output_tar_base}${j}.tar.gz" 
-    echo "creating ${output_tar_file} on node $i" 
-    echo "ssh $i \" cd ${tmp_dir} ;tar -zcf ${output_tar_file} $PACKING\"" 
-    ssh $i " cd ${tmp_dir} ;tar -zcf ${output_tar_file} $PACKING" 
- 
-    echo "copying file back" 
-    mkdir -p ${data_save} 
-    ssh $i cp  ${tmp_dir}\/${output_tar_file} ${data_save} 
-    j=$(echo "$j+1"|bc -l) 
-done 
- 
-# ----------------------------------------------------------------- end-of-file 
- 
- 
-</code> 
  • doku/copy.1620928098.txt.gz
  • Last modified: 2021/05/13 17:48
  • by goldenberg