run on stampede

bin/hdfcoinc/pycbc_page_snrchi

@@ -59,7 +59,7 @@ cb = pylab.colorbar()
 pylab.xlim(snr.min(), snr.max())
 pylab.ylim(chisq.min(), chisq.max())
 cb.set_label('Trigger Density')
-pylab.xlabel('Signal-to-noise Ratio')
+pylab.xlabel('Signal-to-Noise Ratio')
 pylab.ylabel('Reduced $\chi^2$')
 pylab.title('%s: Single Detector Trigger Distribution' % ifo)
 pylab.savefig(args.output_file, dpi=300)

bin/pycbc_submit_dax

@@ -37,9 +37,9 @@ touch _reuse.cache
 
 if [ "x$2" == "x--cache" ]; then
   cp $3 _reuse.cache
-  pegasus-plan --conf $PEG_DIR/basic_pegasus.conf -d $1 --sites local -o local --dir $SUBMIT_DIR --cleanup inplace --cache _reuse.cache --relative-submit-dir work --submit
+  pegasus-plan --conf $PEG_DIR/basic_pegasus.conf -d $1 --sites local -o local --dir $SUBMIT_DIR --cleanup inplace --cache _reuse.cache --relative-submit-dir work --cluster label --submit
 else
-  pegasus-plan --conf $PEG_DIR/basic_pegasus.conf -d $1 --sites local -o local --dir $SUBMIT_DIR --cleanup inplace --relative-submit-dir work --submit
+  pegasus-plan --conf $PEG_DIR/basic_pegasus.conf -d $1 --sites local -o local --dir $SUBMIT_DIR --cleanup inplace --relative-submit-dir work --cluster label --submit 
 fi
 
 ln -s $SUBMIT_DIR submitdir

bin/pycbc_submit_dax_stampede

@@ -0,0 +1,75 @@
+#!/bin/bash
+set -e
+
+# Redirect stdout ( > ) into a named pipe ( >() ) running "tee"
+exec > >(tee submit.log)
+exec 2>&1
+
+# Check I have two arguments supplied
+if [ "x$2" == "x" ]; then
+  echo "I take two arguments, the name of the dax file and the number of hosts to request"
+  exit 1
+fi
+
+# Get the host count
+HOSTS=$2
+
+echo 'making a directory for the submit files'
+export SUBMIT_DIR=`mktemp -d`
+echo $SUBMIT_DIR
+
+#Make sure the directory is world readable
+chmod 755 $SUBMIT_DIR
+
+echo 'finding remote scratch directory'
+export SCRATCH=`gsissh -p 2222 login5.stampede.tacc.utexas.edu 'echo $SCRATCH'`/pycbc_workflow_$RANDOM
+echo $SCRATCH
+
+echo 'finding the remote pegasus home'
+#PEGASUS_HOME=`gsissh -p 2222 login5.stampede.tacc.utexas.edu 'which pegasus_submit_dax'`
+export PEGASUS_HOME=/home1/02969/ahnitz/pegasus-4.5.0
+echo $PEGASUS_HOME
+
+echo 'finding the site-local template directory'
+PEG_DIR=`python -c 'from pycbc.workflow import PEGASUS_FILE_DIRECTORY;print PEGASUS_FILE_DIRECTORY'`
+SITE_TEMPLATE=$PEG_DIR/xsede_site_template.xml
+echo $SITE_TEMPLATE
+
+# Set up the site-local with the correct paths
+echo 'cat <<END_OF_TEXT' >  temp.sh
+cat $SITE_TEMPLATE       >> temp.sh
+echo 'END_OF_TEXT'       >> temp.sh
+bash temp.sh > site-local-parsed.xml
+
+# Plan the workflow
+echo "Generating concrete workflow"
+touch _reuse.cache
+
+if [ "x$3" == "x--cache" ]; then
+  cp $3 _reuse.cache
+  pegasus-plan --conf $PEG_DIR/basic_pegasus.conf -d $1 --sites stampede,local -o local \
+               --dir $SUBMIT_DIR --cleanup inplace --cluster label \
+               --cache _reuse.cache --relative-submit-dir work --submit
+else
+  pegasus-plan --conf $PEG_DIR/basic_pegasus.conf -d $1 --sites stampede,local -o local \
+               --dir $SUBMIT_DIR --cleanup inplace --cluster label --relative-submit-dir work --submit
+fi
+
+ln -s $SUBMIT_DIR submitdir
+
+echo pegasus-status $SUBMIT_DIR/work > status
+chmod 755 status
+
+echo pegasus-analyzer $SUBMIT_DIR/work > debug
+chmod 755 debug
+
+echo pegasus-remove $SUBMIT_DIR/work > stop
+chmod 755 stop
+
+echo pegasus-run $SUBMIT_DIR/work > start
+chmod 755 start
+
+echo 'gsissh -p 2222 login5.stampede.tacc.utexas.edu -t "cd $SCRATCH; bash --login"' > remote_work
+chmod 755 remote_work
+
+echo Submitting from: $HOSTNAME

bin/pycbc_submit_dax_stampede_devel

@@ -0,0 +1,74 @@
+#!/bin/bash
+set -e
+
+# Redirect stdout ( > ) into a named pipe ( >() ) running "tee"
+exec > >(tee submit.log)
+exec 2>&1
+
+# Check I have two arguments supplied
+if [ "x$1" == "x" ]; then
+  echo "I take one argument, the name of the dax file. None supplied."
+  exit 1
+fi
+
+HOSTS=1
+
+echo 'making a directory for the submit files'
+export SUBMIT_DIR=`mktemp -d`
+echo $SUBMIT_DIR
+
+#Make sure the directory is world readable
+chmod 755 $SUBMIT_DIR
+
+echo 'finding remote scratch directory'
+export SCRATCH=`gsissh -p 2222 login5.stampede.tacc.utexas.edu 'echo $SCRATCH'`/pycbc_workflow_$RANDOM
+echo $SCRATCH
+
+echo 'finding the remote pegasus home'
+#PEGASUS_HOME=`gsissh -p 2222 login5.stampede.tacc.utexas.edu 'which pegasus_submit_dax'`
+export PEGASUS_HOME=/home1/02969/ahnitz/pegasus-4.5.0
+echo $PEGASUS_HOME
+
+echo 'finding the site-local template directory'
+PEG_DIR=`python -c 'from pycbc.workflow import PEGASUS_FILE_DIRECTORY;print PEGASUS_FILE_DIRECTORY'`
+SITE_TEMPLATE=$PEG_DIR/xsede_site_template.xml
+echo $SITE_TEMPLATE
+
+# Set up the site-local with the correct paths
+echo 'cat <<END_OF_TEXT' >  temp.sh
+cat $SITE_TEMPLATE       >> temp.sh
+echo 'END_OF_TEXT'       >> temp.sh
+bash temp.sh > site-local-parsed.xml
+
+# Plan the workflow
+echo "Generating concrete workflow"
+touch _reuse.cache
+
+if [ "x$2" == "x--cache" ]; then
+  cp $3 _reuse.cache
+  pegasus-plan --conf $PEG_DIR/basic_pegasus.conf -d $1 --sites stampede-devel,local -o local \
+               --dir $SUBMIT_DIR --cleanup inplace --cluster label \
+               --cache _reuse.cache --relative-submit-dir work --submit
+else
+  pegasus-plan --conf $PEG_DIR/basic_pegasus.conf -d $1 --sites stampede-devel,local -o local \
+               --dir $SUBMIT_DIR --cleanup inplace --cluster label --relative-submit-dir work --submit
+fi
+
+ln -s $SUBMIT_DIR submitdir
+
+echo pegasus-status $SUBMIT_DIR/work > status
+chmod 755 status
+
+echo pegasus-analyzer $SUBMIT_DIR/work > debug
+chmod 755 debug
+
+echo pegasus-remove $SUBMIT_DIR/work > stop
+chmod 755 stop
+
+echo pegasus-run $SUBMIT_DIR/work > start
+chmod 755 start
+
+echo 'gsissh -p 2222 login5.stampede.tacc.utexas.edu -t "cd $SCRATCH; bash --login"' > remote_work
+chmod 755 remote_work
+
+echo Submitting from: $HOSTNAME

pycbc/workflow/core.py

@@ -158,7 +158,13 @@ def __init__(self, cp, name,
             self.tagged_name = name
         if self.ifo_string is not None:
             self.tagged_name = "%s-%s" % (self.tagged_name, self.ifo_string)
-        super(Executable, self).__init__(self.tagged_name)
+
+        try:
+            self.installed = cp.getboolean('pegasus_profile-%s' % name, 'pycbc|installed')
+        except:
+            self.installed = True
+
+        super(Executable, self).__init__(self.tagged_name, installed=self.installed)
 
         self.name=name
 
@@ -293,11 +299,18 @@ def ifo(self):
 
     def add_ini_profile(self, cp, sec):
         for opt in cp.options(sec):
-            value = string.strip(cp.get(sec, opt))
             namespace = opt.split('|')[0]
+            if namespace == 'pycbc':
+                continue
+
+            value = string.strip(cp.get(sec, opt))
             key = opt.split('|')[1]
             self.add_profile(namespace, key, value)
 
+            # Remove if Pegasus can apply this hint in the TC
+            if namespace == 'hints' and key == 'execution.site':
+                self.execution_site = value
+
     def add_ini_opts(self, cp, sec):
         for opt in cp.options(sec):
             value = string.strip(cp.get(sec, opt))
@@ -445,8 +458,12 @@ def __init__(self, executable):
         self.executed = False
         self.set_category(executable.name)
 
-        if executable.universe == 'vanilla':
+        if executable.universe == 'vanilla' and executable.installed:
             self.add_profile('condor', 'getenv', 'True')
+
+        if hasattr(executable, 'execution_site'):
+            self.add_profile('hints', 'execution.site', executable.execution_site)
+            self.add_profile('hints', 'executionPool', executable.execution_site)
 
         self._options += self.executable.common_options
 

pycbc/workflow/pegasus_files/xsede_site_template.xml

@@ -0,0 +1,53 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<sitecatalog xmlns="http://pegasus.isi.edu/schema/sitecatalog" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+xsi:schemaLocation="http://pegasus.isi.edu/schema/sitecatalog http://pegasus.isi.edu/schema/sc-4.0.xsd" version="4.0">
+
+  <site handle="local" arch="x86_64" os="LINUX">
+    <directory  path="$PWD" type="shared-scratch" free-size="null" total-size="null">
+        <file-server  operation="all" url="file://$PWD">
+        </file-server>
+    </directory>
+    <directory  path="$PWD" type="shared-storage" free-size="null" total-size="null">
+        <file-server  operation="all" url="file://$PWD">
+        </file-server>
+    </directory>
+    <profile namespace="env" key="GLOBUS_LOCATION">/usr</profile>
+    <profile namespace="pegasus" key="style">condor</profile>
+    <profile namespace="condor" key="getenv">True</profile>
+    <profile namespace="condor" key="should_transfer_files">YES</profile>
+    <profile namespace="condor" key="when_to_transfer_output">ON_EXIT_OR_EVICT</profile>
+  </site>
+
+    <site handle="stampede-devel" arch="x86_64" os="LINUX">
+        <grid type="gt5" contact="login5.stampede.tacc.utexas.edu:2119/jobmanager-fork" scheduler="Fork" jobtype="auxillary"/>
+        <grid type="gt5" contact="login5.stampede.tacc.utexas.edu:2119/jobmanager-slurm" scheduler="unknown" jobtype="compute"/>
+        <directory type="shared-scratch" path="$SCRATCH">
+            <file-server operation="all" url="gsiftp://gridftp.stampede.tacc.xsede.org/$SCRATCH"/>
+        </directory>
+        <profile namespace="env" key="PEGASUS_HOME">$PEGASUS_HOME</profile>
+        <profile namespace="pegasus" key="job.aggregator" >mpiexec</profile>
+        <profile namespace="globus" key="queue">development</profile>
+        <profile namespace="globus" key="maxwalltime">1</profile>
+        <profile namespace="globus" key="host_count">1</profile>
+        <profile namespace="globus" key="count">16</profile>
+        <profile namespace="globus" key="jobtype">single</profile>
+        <profile namespace="globus" key="project">TG-PHY140012</profile>
+    </site>
+
+    <site handle="stampede" arch="x86_64" os="LINUX">
+        <grid type="gt5" contact="login5.stampede.tacc.utexas.edu:2119/jobmanager-fork" scheduler="Fork" jobtype="auxillary"/>
+        <grid type="gt5" contact="login5.stampede.tacc.utexas.edu:2119/jobmanager-slurm" scheduler="unknown" jobtype="compute"/>
+        <directory type="shared-scratch" path="$SCRATCH">
+            <file-server operation="all" url="gsiftp://gridftp.stampede.tacc.xsede.org/$SCRATCH"/>
+        </directory>
+        <profile namespace="env" key="PEGASUS_HOME">$PEGASUS_HOME</profile>
+        <profile namespace="pegasus" key="job.aggregator" >mpiexec</profile>
+        <profile namespace="globus" key="queue">normal</profile>
+        <profile namespace="globus" key="maxwalltime">30</profile>
+        <profile namespace="globus" key="host_count">$HOSTS</profile>
+        <profile namespace="globus" key="count">16</profile>
+        <profile namespace="globus" key="jobtype">single</profile>
+        <profile namespace="globus" key="project">TG-PHY140012</profile>
+    </site>
+
+</sitecatalog>

setup.py

@@ -338,6 +338,8 @@ def run(self):
                'bin/sngl/pycbc_plot_timeseries',
                'bin/hdfcoinc/pycbc_page_injtable',
                'bin/pycbc_submit_dax',
+               'bin/pycbc_submit_dax_stampede',
+               'bin/pycbc_submit_dax_stampede_devel',
                'bin/hdfcoinc/pycbc_page_coinc_snrchi',
                'bin/hdfcoinc/pycbc_stat_dtphase',
                ],