Autosix python2 update (#61)

Fixes #62

* Added python2 input parameter
* bugfix with duplicated column names
* limit pandas version <1.2 because of pandas-dev/pandas#39872
* updated changelog

Author: JoschD

.zenodo.json

@@ -3,6 +3,30 @@
         {
             "name": "OMC-Team",
             "affiliation": "CERN"
+        },
+        {
+            "name": "Michael Hofer",
+            "affiliation": "CERN",
+            "orcid": "0000-0001-6173-0232"
+        },
+        {
+            "name": "Joschua Dilly",
+            "affiliation": "CERN",
+            "orcid": "0000-0001-7864-5448"
+        },
+        {
+            "name": "Felix Soubelet",
+            "affiliation": "University of Liverpool & CERN",
+            "orcid": "0000-0001-8012-1440"
+        },
+        {
+            "name": "Rogelio Tomas Garcia",
+            "affiliation": "CERN",
+            "orcid": "0000-0002-9857-1703"
+        },
+        {
+            "name": "Tobias Persson",
+            "affiliation": "CERN"
         }
     ],
     "title": "PyLHC",

CHANGELOG.md

@@ -1,5 +1,13 @@
 # pylhc Changelog
 
+## Version 0.1.1
+
+- Added:
+    - `python2` parameter for autosix.
+  
+- Changed:
+    - Bugfix for non-unique column names when indexing (`forced_da_analysis`)
+
 ## Version 0.1.0
 
 - Added:

pylhc/__init__.py

@@ -10,7 +10,7 @@
 __title__ = "pylhc"
 __description__ = "An accelerator physics script collection for the OMC team at CERN."
 __url__ = "https://github.com/pylhc/pylhc"
-__version__ = "0.1.0"
+__version__ = "0.1.1"
 __author__ = "pylhc"
 __author_email__ = "[email protected]"
 __license__ = "MIT"

pylhc/autosix.py

@@ -204,14 +204,23 @@
     Mask to name jobs from replace_dict
 
 
-- **python** *(PathOrStr)*:
+- **python3** *(PathOrStr)*:
 
     Path to python to use with sixdb (python3 with requirements
     installed).
 
     default: ``python3``
 
 
+- **python2** *(PathOrStr)*:
+
+    Path to python to use with run_six.sh (python2 with requirements installed).
+    ONLY THE PATH TO THE DIRECTORY OF THE python BINARY IS NEEDED!
+    And it can't be an Anaconda Distribution.
+
+    default: None (uses the first `python` in path)
+
+
 - **resubmit**:
 
     Resubmits if needed.
@@ -321,8 +330,16 @@ def get_params():
         help="Path to executable.",
     )
     params.add_parameter(
-        name="python",
-        default=DEFAULTS["python"],
+        name="python2",
+        default=DEFAULTS["python2"],
+        type=PathOrStr,
+        help=("Path to python to use with run_six.sh (python2 with requirements installed)."
+              " ONLY THE PATH TO THE DIRECTORY OF THE python BINARY IS NEEDED!"
+              " And it can't be an Anaconda Distribution."),
+    )
+    params.add_parameter(
+        name="python3",
+        default=DEFAULTS["python3"],
         type=PathOrStr,
         help="Path to python to use with sixdb (python3 with requirements installed).",
     )
@@ -388,7 +405,8 @@ def main(opt):
             basedir=opt.working_directory,
             # kwargs:
             ssh=opt.ssh,
-            python=opt.python,
+            python2=opt.python2,
+            python3=opt.python3,
             unlock=opt.unlock,
             resubmit=opt.resubmit,
             da_turnstep=opt.da_turnstep,
@@ -426,7 +444,8 @@ def setup_and_run(jobname: str, basedir: Path, **kwargs):
     LOG.info(f"vv---------------- Job {jobname} -------------------vv")
     unlock: bool = kwargs.pop("unlock", False)
     ssh: str = kwargs.pop("ssh", None)
-    python: Union[Path, str] = kwargs.pop("python", DEFAULTS["python"])
+    python2: Union[Path, str] = kwargs.pop("python2", DEFAULTS["python2"])
+    python3: Union[Path, str] = kwargs.pop("python3", DEFAULTS["python3"])
     resubmit: bool = kwargs.pop("resubmit", False)
     da_turnstep: int = kwargs.pop("da_turnstep", DEFAULTS["da_turnstep"])
     ignore_twissfail_check: bool = kwargs.pop("ignore_twissfail_check", False)
@@ -499,7 +518,7 @@ def setup_and_run(jobname: str, basedir: Path, **kwargs):
         > /afs/cern.ch/project/sixtrack/SixDesk_utilities/pro/utilities/bash/run_six.sh -a
         """
         if check_ok:
-            submit_sixtrack(jobname, basedir, ssh=ssh)
+            submit_sixtrack(jobname, basedir, python=python2, ssh=ssh)
             return  # takes even longer
 
     with check_stage(STAGES.check_sixtrack_output, jobname, basedir) as check_ok:
@@ -515,7 +534,7 @@ def setup_and_run(jobname: str, basedir: Path, **kwargs):
         > /afs/cern.ch/project/sixtrack/SixDesk_utilities/pro/utilities/bash/run_six.sh -i
         """
         if check_ok:
-            check_sixtrack_output(jobname, basedir, ssh=ssh, resubmit=resubmit)
+            check_sixtrack_output(jobname, basedir, python=python2, ssh=ssh, resubmit=resubmit)
 
     with check_stage(STAGES.sixdb_load, jobname, basedir) as check_ok:
         """
@@ -524,7 +543,7 @@ def setup_and_run(jobname: str, basedir: Path, **kwargs):
         > python3 /afs/cern.ch/project/sixtrack/SixDesk_utilities/pro/utilities/externals/SixDeskDB/sixdb . load_dir
         """
         if check_ok:
-            sixdb_load(jobname, basedir, python=python, ssh=ssh)
+            sixdb_load(jobname, basedir, python=python3, ssh=ssh)
 
     with check_stage(STAGES.sixdb_cmd, jobname, basedir) as check_ok:
         """
@@ -537,7 +556,7 @@ def setup_and_run(jobname: str, basedir: Path, **kwargs):
         > python3 /afs/cern.ch/project/sixtrack/SixDesk_utilities/pro/utilities/externals/SixDeskDB/sixdb $jobname plot_da_vs_turns
         """
         if check_ok:
-            sixdb_cmd(jobname, basedir, cmd=["da"], python=python, ssh=ssh)
+            sixdb_cmd(jobname, basedir, cmd=["da"], python=python3, ssh=ssh)
 
             # da_vs_turns is broken at the moment (jdilly, 19.10.2020)
             # sixdb_cmd(jobname, basedir, cmd=['da_vs_turns', '-turnstep', str(da_turnstep), '-outfile'],

pylhc/constants/autosix.py

@@ -28,7 +28,8 @@
 HEADER_BASEDIR = "BASEDIR"
 
 DEFAULTS = dict(
-    python="python3",
+    python2=None,
+    python3="python3",
     da_turnstep=100,
     executable=MADX_BIN,
 )

pylhc/data_extract/timber.py

@@ -18,7 +18,7 @@ def find_exact_time_for_beamprocess(acc_time: AccDatetime) -> AccDatetime:
     KnobExtractor, the timing event for SQUEEZE or RAMP.
     I don't think this is correct (jdilly).
     """
-    db = pytimber.LoggingDB()
+    db = pytimber.LoggingDB(source="nxcals")
     t1, t2 = acc_time.sub(days=1).local_string(), acc_time.local_string()
 
     event_ts, event_val = db.get("HX:SRMP-POW", t1, t2)["HX:SRMP-POW"]

pylhc/forced_da_analysis.py

@@ -753,7 +753,7 @@ def _get_old_kick_file(kick_dir, plane):
     df = df.set_index(TIME_COLUMN)
     df.index = _convert_time_index(df.index, path)
     rename_dict = {}
-    for p in plane:
+    for p in plane:  # can be XY
         rename_dict.update(
             {
                 f"2J{p}RES": column_action(p),
@@ -763,7 +763,8 @@ def _get_old_kick_file(kick_dir, plane):
             }
         )
     df = df.rename(rename_dict, axis="columns")
-    df.loc[:, rename_dict.values()] = df.loc[:, rename_dict.values()] * 1e-6
+    renamed_cols = list(set(rename_dict.values()))
+    df.loc[:, renamed_cols] = df.loc[:, renamed_cols] * 1e-6
     return df
 
 

pylhc/sixdesk_tools/submit.py

@@ -49,13 +49,17 @@ def check_sixtrack_input(jobname: str, basedir: Path, ssh: str = None, resubmit:
         LOG.info("Check for input files was successful.")
 
 
-def submit_sixtrack(jobname: str, basedir: Path, ssh: str = None, resubmit: bool = False):
+def submit_sixtrack(jobname: str, basedir: Path, python: Path = None, ssh: str = None, resubmit: bool = False):
     """ Generate simulation files and check if runnable and submit. """
     re_str = "Re-" if resubmit else ""
     LOG.info(f"{re_str}Submitting to sixtrack.")
     sixjobs_path = get_sixjobs_path(jobname, basedir)
+    args = ["-i"] if resubmit else ["-a"]
+    if python is not None:
+        if not python.is_dir():
+            python = python.parent
+        args += ["-P", str(python)]
     try:
-        args = ["-i"] if resubmit else ["-a"]
         start_subprocess([RUNSIX_SH] + args, cwd=sixjobs_path, ssh=ssh)  # throws OSError if failed
     except OSError as e:
         raise StageSkip(
@@ -66,20 +70,24 @@ def submit_sixtrack(jobname: str, basedir: Path, ssh: str = None, resubmit: bool
         LOG.info(f"{re_str}Submitted jobs to Sixtrack")
 
 
-def check_sixtrack_output(jobname: str, basedir: Path, ssh: str = None, resubmit: bool = False):
+def check_sixtrack_output(jobname: str, basedir: Path, python: Union[Path, str], ssh: str = None, resubmit: bool = False):
     """ Checks if the sixtrack output is all there. """
     LOG.info("Checking if sixtrack has finished.")
     sixjobs_path = get_sixjobs_path(jobname, basedir)
     try:
         start_subprocess([RUNSTATUS_SH], cwd=sixjobs_path, ssh=ssh)
     except OSError as e:
         if resubmit:
-            submit_sixtrack(jobname, basedir, ssh, resubmit=True)
-            raise StageSkip("Resubmitted incomplete sixtrack jobs.")
+            submit_sixtrack(jobname, basedir, python=python, ssh=ssh, resubmit=True)
+            raise StageSkip(
+                f"Sixtrack for {jobname} seems to be incomplete."
+                " Resubmitted incomplete sixtrack jobs."
+                " Wait until they have finished and run again."
+            )
         else:
             raise StageSkip(
                 f"Sixtrack for {jobname} seems to be incomplete."
-                f"Run possibly not finished. Check (debug-) log or your Scheduler."
+                f" Run possibly not finished. Check (debug-) log or your Scheduler."
             ) from e
     else:
         LOG.info("Sixtrack results are all present.")

setup.py

@@ -31,7 +31,7 @@ def about_package(init_posixpath: pathlib.Path) -> dict:
 DEPENDENCIES = [
     "numpy>=1.19",
     "scipy>=1.4.0",
-    "pandas>=1.0",
+    "pandas>=1.0,<1.2",  # limit because of https://github.com/pandas-dev/pandas/issues/39872
     "matplotlib>=3.2.0",
     "pjlsa>=0.0.14",
     "pytimber>=2.8.0",