Skip to content

Commit cdf57d7

Browse files
shalooshaloo
committed
Enhanced scripts to generate numpy dependency graph for various pkg
1 parent b7a17df commit cdf57d7

File tree

4 files changed

+358
-0
lines changed

4 files changed

+358
-0
lines changed

‎scripts/cleanup-numpydeppkg.sh

+64
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,64 @@
1+
#!/bin/bash
2+
# Script to cleanup and uninstall
3+
# python package fetched using fetch-numpydeppkg.sh script
4+
#
5+
6+
set -ue
7+
8+
cleanup_pkg() {
9+
10+
pkgname=$1
11+
workingdir=$2
12+
setupdir=$3
13+
14+
if [ -z "$pkgname" ] || [ -z "$workingdir" ] || [ -z "$setupdir" ]; then
15+
display_help
16+
exit
17+
fi
18+
19+
ygraphviz=`$workingdir/env-${pkgname}/bin/pip3 list | grep graphviz`
20+
if ! [ -z "$ygraphviz" ];
21+
then
22+
$workingdir/env-${pkgname}/bin/pip3 uninstall -y graphviz
23+
fi
24+
25+
ypipdeptree=`$workingdir/env-${pkgname}/bin/pip3 list | grep pipdeptree`
26+
if ! [ -z "$ypipdeptree" ];
27+
then
28+
$workingdir/env-${pkgname}/bin/pip3 uninstall -y pipdeptree
29+
fi
30+
31+
$workingdir/env-${pkgname}/bin/pip3 uninstall -y -r $workingdir/numpy_${pkgname}_dep/$setupdir/requirements.txt
32+
33+
return 0
34+
}
35+
36+
display_help() {
37+
echo "-----------------------------------------------------------------------"
38+
echo "Usage:"
39+
echo ""
40+
echo "./cleanup-numpydeppkg.sh pkgname workingdir setupdirname"
41+
echo ""
42+
echo "where:"
43+
echo ""
44+
echo "pkgname - the name of package in pypi directory"
45+
echo "workingdir - the directory where latest sources are located"
46+
echo "setupdirname - the name of top level source directory that contains
47+
setup.py for the package."
48+
echo ""
49+
echo "For example, to download ehtim use the following command:"
50+
echo ""
51+
echo "./cleanup-numpydeppkg.sh <pkgname> <working-dir> <setupdirname>"
52+
echo ""
53+
echo "-----------------------------------------------------------------------"
54+
}
55+
56+
if [[ ( $# -gt 3) || ( -z "$1" )|| ( -z "$2" ) || ( -z "$3" ) || ("$1" == "-h" ) || ("$1" == "--help") ]];
57+
then
58+
display_help
59+
exit
60+
fi
61+
62+
cleanup_pkg $1 $2 $3
63+
64+
echo Done!

‎scripts/fetch-numpydeppkg.sh

+97
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,97 @@
1+
#!/bin/bash
2+
# Script to fetch and install python package that depends on numpy
3+
# This script has been tested for eht-imaging, gwpy and pycbc packages
4+
# For more details on eht-imaging refer to the links below:
5+
# https://github.com/achael/eht-imaging
6+
# https://achael.github.io/_pages/imaging/
7+
# For more details on gwpy see: https://github.com/gwpy/gwpy
8+
# For more details on pycbc see: https://github.com/gwastro/pycbc
9+
10+
# The following function does the following:
11+
# 1. Fetches latest software sources for the specified package in the given working dir
12+
# (default current-dir/numpy_<pkgname>_dep)
13+
# 2. Assumes python3, virtualenv, python and pip3 are available and installs graphviz, pipdeptree
14+
# 3. Git clones specified package
15+
#
16+
# To generate the dependency graph for the specified package and highlight the
17+
# role of NumPy use the script gen-numpy-dep-graph.sh
18+
#
19+
20+
get_latest_pkg() {
21+
22+
pkgname=$1
23+
nowdir=`pwd`
24+
workingdir=$2
25+
cd $workingdir
26+
27+
if [ -d numpy_${pkgname}_dep ];
28+
then
29+
\rm -rf numpy_${pkgname}_dep
30+
fi
31+
mkdir numpy_${pkgname}_dep
32+
cd numpy_${pkgname}_dep
33+
34+
virtualenv --python=python3 $workingdir/env-${pkgname}
35+
source $workingdir/env-${pkgname}/bin/activate
36+
37+
ygraphviz=`$workingdir/env-${pkgname}/bin/pip3 list | grep graphviz`
38+
if [ -z "$ygraphviz" ];
39+
then
40+
$workingdir/env-${pkgname}/bin/pip3 install graphviz
41+
fi
42+
43+
ypipdeptree=`$workingdir/env-${pkgname}/bin/pip3 list | grep pipdeptree`
44+
if [ -z "$ypipdeptree" ];
45+
then
46+
$workingdir/env-${pkgname}/bin/pip3 install pipdeptree
47+
fi
48+
49+
git clone $3
50+
cd $4
51+
$workingdir/env-${pkgname}/bin/pip3 install .
52+
53+
deactivate
54+
cd $nowdir
55+
return 0
56+
}
57+
58+
display_help() {
59+
echo "-----------------------------------------------------------------------"
60+
echo "Usage:"
61+
echo ""
62+
echo "./fetch-numpydeppkg.sh pkgname workingdir github_srcurl setupdirname"
63+
echo ""
64+
echo "where:"
65+
echo ""
66+
echo "pkgname - the name of package in pypi directory"
67+
echo "workingdir - the directory where latest sources will be git cloned and
68+
built"
69+
echo "github_srcurl - the github link to project sources for git cloning"
70+
echo "setupdirname - the name of top level source directory that contains"
71+
echo "setup.py for the package."
72+
echo ""
73+
echo "For example, to download ehtim use the following command:"
74+
echo ""
75+
echo "./fetch-numpydeppkg.sh pkgname <working-dir>"
76+
echo " https://github.com/achael/eht-imaging.git eht-imaging"
77+
echo ""
78+
echo "-----------------------------------------------------------------------"
79+
}
80+
81+
curdir=`pwd`
82+
83+
if [[ ( "$#" -gt "4") || ( -z "$1" )|| ( -z "$3" ) || ( -z "$4" ) || ("$1" == "-h" ) || ("$1" == "--help") ]];
84+
then
85+
display_help
86+
exit
87+
fi
88+
89+
if [ ! -d "$2" ];
90+
then
91+
echo "Error: $2 is not a valid directory!!!"
92+
exit
93+
fi
94+
95+
get_latest_pkg $1 $2 $3 $4
96+
97+
echo Done!

‎scripts/gen-numpy-dep-graph.sh

+123
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,123 @@
1+
#!/bin/bash
2+
3+
# gen-numpy-dep-graph.sh script is used to generate numpy dependency
4+
# graph for a given package. If you already don't have the package
5+
# sources available locally, you can use the script get-package
6+
# to download package sources before running this script.
7+
# The full dependency graph is stored as numpy-<pkgname>-dep.png file
8+
# in the specified output directory. Besides the full chart, for simplicity
9+
# this script trims specified package project dependency graph in dot format
10+
# Trimming is done such that only NumPy oriented dependencies are in focus
11+
12+
# This script along wiht get-package.sh script has been used to create
13+
# NumPy dependency graphs for ehtim-imaging package, gwpy and cbPy packages.
14+
# It should work fine for any specified package that depends on NumPy.
15+
# Feedback is most welcome.
16+
17+
# For more details on eht-imaging project visit: https://github.com/achael/eht-imaging
18+
# For more details on gwpy project visit: https://github.com/gwpy/gwpy
19+
20+
display_help() {
21+
echo "-----------------------------------------------------------------------"
22+
echo "Usage:"
23+
echo " gen-numpy-dep-graph.sh pkgname workingdir reqfilepath outputdir [optional color]"
24+
echo ""
25+
echo "pkgname - the name of the package that uses numpy, use the same name
26+
echo "workingdir - the workingdir pathname where pkgname sources are available"
27+
that you specify to pip3 to install the package."
28+
echo "reqfilepath - directory containing latest source of eht-imaging requirements.txt file."
29+
echo "outputdir - dir or path where generated graphs will be stored."
30+
echo "optional color. For e.g., cyan, crimson or any of the https://graphviz.gitlab.io/_pages/doc/info/colors.html specifications for graphviz. Default cyan."
31+
echo ""
32+
echo "-----------------------------------------------------------------------"
33+
}
34+
35+
check_pre_requisites() {
36+
pkgname=$1
37+
vdir="$2/env-$pkgname/bin"
38+
ypkgname=`$vdir/pip3 list | grep "${pkgname}"`
39+
ygraphviz=`$vdir/pip3 list | grep graphviz`
40+
ypipdeptree=`$vdir/pip3 list | grep pipdeptree`
41+
if [[ ( -z "$ypkgname" ) || ( -z "$ygraphviz" ) || ( -z "$ypipdeptree" ) ]];
42+
then
43+
echo "Error: Pre-requisite for generating dependency graph not fulfilled!"
44+
echo "The following packages must be installed:"
45+
echo " 1. $pkgname"
46+
echo " 2. graphviz"
47+
echo " 3. pipdeptree"
48+
exit
49+
fi
50+
}
51+
52+
if [[ ($# -lt 3) || ($# -gt 5 ) || ( -z "$1" ) || (-z "$2") || (-z "$3") || ("$1" == "-h") || ("$1" == "--help") ]];
53+
then
54+
display_help
55+
exit
56+
fi
57+
58+
inputpkgname=$1
59+
venvdir=$2
60+
reqfilepath=$3
61+
reqfile="$3/requirements.txt"
62+
if ! [ -f $reqfile ];
63+
then
64+
echo "Error: Make sure you have latest $inputpkgname sources and requirements.txt file path is correct!!"
65+
display_help
66+
exit
67+
fi
68+
69+
graphdir="$4"
70+
if [[ ( ! -d "$graphdir" ) || ( -f "$graphdir" ) ]];
71+
then
72+
echo "Error: Make sure $graphdir is a valid directory to store generated graphs!"
73+
display_help
74+
exit
75+
fi
76+
77+
check_pre_requisites $inputpkgname $venvdir
78+
echo "Using $reqfile to prune dependency graph of $inputpkgname."
79+
echo "Specified output directory is $graphdir."
80+
echo "Generating $inputpkgname dependency graphs..."
81+
82+
inputfile=$graphdir/numpy-${inputpkgname}-dep.dot
83+
outputfile=$graphdir/numpy-${inputpkgname}-dep.png
84+
$venvdir/env-${inputpkgname}/bin/pipdeptree --graph-output dot > $graphdir/numpy-${inputpkgname}-dep.dot
85+
dot -T png $inputfile -o $outputfile
86+
87+
awk '/^[0-9]*[.]+/ {printf(" %s\n", $0); found=1; next} !/^[0-9]*[.]+/ {printf((found==1)? "%s" : "\n%s", $0); found=0 } END {print ""}' $inputfile > $graphdir/cleanup.dot
88+
89+
echo "Cleaning up requirements file...."
90+
reqfiletmp=$graphdir/tmpfile
91+
reqfiletmp1=$graphdir/tmpfile1
92+
93+
sed '/^#/ d' $reqfile > $reqfiletmp
94+
sed '/^$/ d' $reqfiletmp > $reqfiletmp1
95+
awk -F '<|=|>|;' '{print $1}' $reqfiletmp1 > $graphdir/reqfileclean
96+
97+
echo digraph { > $graphdir/cleanup_pkg.dot
98+
99+
awk ' NR==FNR{a[$1]; next} {for (i in a) if ((index($0, i)&& $3 == "") || ($3 != "" && index($0, i) && index($3, i) )) print}' $graphdir/reqfileclean $graphdir/cleanup.dot >> $graphdir/cleanup_pkg.dot
100+
101+
echo "Cleaning up temp files"
102+
\rm -rf $graphdir/tmpfile $graphdir/tmpfile1 $graphdir/reqfileclean
103+
echo } >> $graphdir/cleanup_pkg.dot
104+
105+
dot -K twopi $graphdir/cleanup_pkg.dot -o $graphdir/numpy-clean.dot
106+
dot -T png $graphdir/numpy-clean.dot -o $graphdir/numpy-clean.png
107+
108+
if [ -z "$5" ];
109+
then
110+
#hcolor="crimson"
111+
hcolor="cyan"
112+
else
113+
hcolor="$5"
114+
fi
115+
116+
echo "Highlighting numpy nodes using $hcolor."
117+
118+
awk -v ac=${hcolor} '{ if ($1 == "numpy") {print $0, "color=",ac,", style=filled," } else { if ($2 == "->" && $3 == "numpy") {print $0, "color=",ac,","} else {print}}}' $graphdir/numpy-clean.dot > $graphdir/numpy-clean-color.dot
119+
120+
dot -T png $graphdir/numpy-clean-color.dot -o $graphdir/numpy-clean-color.png
121+
122+
echo "-------------------------------------------------------------------"
123+
echo Done!

‎scripts/redraw-numpy-dep-charts.sh

+74
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,74 @@
1+
#!/bin/bash
2+
# This script refreshes all numpy dependency graphs for packages
3+
# that are mentioned in NumPy case studes: ehtim, gwpy and PyCBC
4+
#
5+
6+
set -ue
7+
8+
curdir=`pwd`
9+
outputdir=$curdir
10+
11+
if [[ "$#" -ge "2" || (( "$#" -eq "1" ) && ( "$1" == "-h" || "$1" == "--help" )) ]]; then
12+
echo ""
13+
echo "Usage: ./redraw-numpy-dep-charts.sh outputdirname <optional, default current-dir"
14+
echo ""
15+
exit
16+
fi
17+
18+
if [[ $# -eq "0" ]]; then
19+
echo "Warning: No output dir specified, using $curdir as default output dir."
20+
elif [[ $# -eq "1" ]]; then
21+
outputdir=$1
22+
fi
23+
24+
if ! [[ -d $outputdir ]]; then
25+
echo "Warning: Specified output directory: $outputdir does not exist!!!"
26+
exit
27+
fi
28+
29+
# Using /tmp/numpy and the package source processing dir for dependency graphs
30+
31+
workingdir="/tmp/numpy"
32+
if [ -d $workingdir ];
33+
then
34+
\rm -rf $workingdir
35+
fi
36+
mkdir $workingdir
37+
38+
declare -a pkgarray=("ehtim" "gwpy" "PyCBC")
39+
declare -a pkgurl=("https://github.com/achael/eht-imaging.git"
40+
"https://github.com/gwpy/gwpy.git"
41+
"https://github.com/gwastro/pycbc.git")
42+
declare -a pkgsetup=("eht-imaging" "gwpy" "pycbc")
43+
highlightcolor="cyan"
44+
45+
#numpkg=${#pkgarray[@]}
46+
47+
for index in "${!pkgarray[@]}" ;
48+
do
49+
echo ".....Drawing ${pkgarray[$index]} dependency chart....."
50+
echo "Using....pkgurl = ${pkgurl[$index]}"
51+
echo "installing pkg from dir pkgsetup = ${pkgsetup[$index]}"
52+
echo "and using $workingdir as scratch"
53+
54+
mkdir $workingdir/${pkgarray[$index]} $workingdir/${pkgarray[$index]}/graphdir
55+
./scripts/fetch-numpydeppkg.sh ${pkgarray[$index]} $workingdir/${pkgarray[$index]} ${pkgurl[$index]} ${pkgsetup[$index]}
56+
./scripts/gen-numpy-dep-graph.sh ${pkgarray[$index]} $workingdir/${pkgarray[$index]} $workingdir/${pkgarray[$index]}/numpy_${pkgarray[$index]}_dep/${pkgsetup[$index]} $workingdir/${pkgarray[$index]}/graphdir/ $highlightcolor
57+
if ! [ -f $workingdir/${pkgarray[$index]}/graphdir/numpy-clean-color.png ]
58+
then
59+
echo "Error: Graph for ${pkgarray[$index]} failed! Exiting."
60+
exit
61+
fi
62+
cp $workingdir/${pkgarray[$index]}/graphdir/numpy-clean-color.png $outputdir/${pkgarray[$index]}-numpy-dep-graph.png
63+
done
64+
65+
#Clean up
66+
for index in "${!pkgarray[@]}" ;
67+
do
68+
echo "Uninstalling ${pkgarray[$index]}....."
69+
./scripts/cleanup-numpydeppkg.sh ${pkgarray[$index]} $workingdir/${pkgarray[$index]} ${pkgsetup[$index]}
70+
done
71+
72+
echo "Cleaning up scratch dir: $workingdir..........."
73+
\rm -rf $workingdir
74+
echo "Done!"

0 commit comments

Comments
 (0)