Merge pull request #10 from rkkautsar/refactor

Implement yaml for runscript encoding

Author: rkkautsar

.gitmodules

@@ -1,3 +1,5 @@
-[submodule "submodules/runsolver"]
-	path = submodules/runsolver
+[submodule "external-tools/runsolver"]
+	path = external-tools/runsolver
 	url = [email protected]:daajoe/runsolver.git
+	ignore = dirty
+

Pipfile

@@ -8,6 +8,9 @@ verify_ssl = true
 [packages]
 lxml = "*"
 jinja2 = "*"
+pyyaml = "*"
+pykwalify = "*"
+dotmap = "*"
 
 [requires]
 python_version = "3"

Pipfile.lock

@@ -1,6 +1,9 @@
 *
 
 !*/
-!examples/*
 !.gitignore
 
+!examples/*
+!examples/**/*
+examples/**/output/
+

benchmarks/.gitignore

@@ -0,0 +1,111 @@
+base_dir: benchmarks/examples/clasp
+output_dir: output
+
+machines:
+  houat:
+    cpu: [email protected]
+    memory: 24GB
+  zuse:
+    cpu: [email protected]
+    memory: 24GB
+
+configs:
+  seq-generic:
+    template: templates/seq-generic.sh
+  pbs-generic:
+    template: templates/pbs-generic.sh
+
+systems:
+  - name: clasp
+    version: 1.3.2
+    measures: .resultparsers.clasp
+    config: seq-generic
+    settings:
+      - name: default
+        tag: seq
+        cmdline: '--stats 1'
+      - name: vsids
+        tag: seq
+        cmdline: '--stats --heu=vsids 1'
+      - name: progress
+        tag: seq
+        cmdline: '--stats --save-progress 1'
+      - name: estimate
+        tag: seq
+        cmdline: '--stats --estimate 1'
+      - name: recstr
+        tag: seq
+        cmdline: '--stats --recursive-str 1'
+      - name: nodel
+        tag: seq
+        cmdline: '--stats --del=no 1'
+      - name: norest
+        tag: seq
+        cmdline: '--stats --restarts=no 1'
+  - name: claspar
+    version: 2.1.0
+    measures: .resultparsers.claspar
+    config: pbs-generic
+    settings:
+      - name: one-as
+        tag: par one-as
+        cmdline: '--stats 1'
+        ppn: 2
+        procs: 1 2 4 8
+        pbstemplate: templates/impi.pbs
+      - name: all-as
+        tag: par all-as
+        cmdline: '--stats -q 0'
+        ppn: 2
+        procs: 1 2 4 8
+        pbstemplate: templates/impi.pbs
+
+jobs:
+  seq-generic:
+    type: seq
+    timeout: 120
+    runs: 1
+    parallel: 8
+  pbs-generic:
+    type: pbs
+    timeout: 120
+    runs: 1
+    script_mode: timeout
+    walltime: 23:59:59
+
+benchmarks:
+  seq-suite:
+    - type: folder
+      path: instances
+      ignore_prefixes:
+        - pigeons
+    - type: files
+      path: instances
+      files:
+        - pigeons/pigeonhole10-unsat.lp
+        - pigeons/pigeonhole11-unsat.lp
+  pbs-suite:
+    - type: folder
+      path: instances
+
+projects:
+  clasp-big:
+    job: seq-generic
+    runtags:
+      - machine: houat
+        benchmark: seq-suite
+        tag: seq
+  claspar-all-as:
+    job: pbs-generic
+    runtags:
+      - machine: houat
+        benchmark: pbs-suite
+        tag: par all-as
+  claspar-one-as:
+    job: pbs-generic
+    runspecs:
+      - machine: zuse
+        benchmark: pbs-suite
+        system: claspar
+        version: 2.1.0
+        setting: one-as

benchmarks/examples/clasp/runscript-all.xml

@@ -47,7 +47,8 @@ def sudokuresultparser(root, runspec, instance):
         'run': ('int', 0),
         'objective': ('int', 0),
         'error': ('int', 0),
-        'status': ('int', -1)
+        'status': ('int', -1),
+        'error_str': ('str', ''),
     }
 
     instance_str = instance.instance
@@ -122,7 +123,7 @@ def sudokuresultparser(root, runspec, instance):
     try:
         instance_output = codecs.open(os.path.join(root, 'runsolver.solver'), encoding='utf-8').read()
         output_file = os.path.join(root, 'runsolver.solver')
-        validator = 'benchmarks/sudokusat2019/bin/sudoku_validate-1.0'
+        validator = 'benchmarks/examples/sudokusat2019/bin/sudoku_validate-1.0'
         validator_output = subprocess.check_output("./%s -f %s" % (validator, output_file), shell=True, encoding='utf-8')
     except IOError:
         sys.stderr.write('Instance %s did not finish properly. Missing output file.\n' % root)