Merge commit '3e01e4d5fdfa1a0ca65704d7d7f0ae8a49f5dc96' into release/graal-vm/1.0

Author: ansalond

CHANGELOG.md

@@ -3,6 +3,28 @@
 This changelog summarizes major changes between GraalVM versions of the Python
 language runtime. The main focus is on user-observable behavior of the engine.
 
+## Version 1.0.0 RC11
+
+* Support running setuptools to build and install various packages
+* Support running a source release version of NumPy out of the box
+* Improve performance of member access to C API objects
+* Improve performance of binary operations on C API objects
+* Add support for `yield from`
+* Support assignment to `object.__dict__` and ensure that managed subclasses of native types also have a `__dict__`
+* Fix `[]` access with non-integer keys for array-like foreign objects
+* Fix various performance regressions introduced in the last RC
+* Implement more built-in methods on the `time` module
+* Python no longer exposes internal languages through `polyglot.eval`
+* Improve performance of `os.scandir` and functions that build on it (such as `glob`)
+* More correct implementation of standard streams, including buffering
+* Properly support the `-m` switch to run modules
+* Support the standard `zipfile` module
+* Add the built-in `_cvs` module
+* Add support for `__slots__`
+* Allow arbitrary callable objects in methods, not only functions
+* Report that we have a TTY console if we are launched on a Terminal through our launcher
+* Add the `ginstall` custom module to install known packages such as NumPy and setuptools
+
 ## Version 1.0.0 RC10
 
 * Improve performance of C API upcalls

README.md

@@ -14,6 +14,28 @@ To try it, you can use the bundled releases from
 some examples of what you can do with it, check out the
 [reference](https://www.graalvm.org/docs/reference-manual/languages/python/).
 
+### Installing packages
+
+At the moment not enough of the standard library is implemented to run the
+standard package installers for many packages. As a convenience, we provide a
+simple module to install packages that we know to be working (including
+potential patches required for those packages). Try the following to find out
+more:
+
+```
+graalpython -m ginstall --help
+```
+
+As a slightly more exciting example, try:
+
+```
+graalpython -m ginstall install numpy
+```
+
+If all goes well (you'll need to have `clang`, `llvm-link`, `llvm-extract`,
+`llvm-nm`, and `opt` in your `PATH` in addition to the normal NumPy build
+dependencies), you should be able to `import numpy` afterwards.
+
 ### Licensing
 
 This Graal/Truffle-based implementation of Python is copyright (c) 2017, 2018
@@ -23,3 +45,4 @@ Universal Permissive License v 1.0 as shown at
 implementation is in part derived from and contains additional code from 3rd
 parties, the copyrights and licensing of which is detailed in the
 [LICENSE](LICENSE) and [3rd_party_licenses.txt](3rd_party_licenses.txt) files.
+

ci.jsonnet

@@ -1,5 +1,5 @@
 {
-  overlay: "ffde557d62fff381069838b745c31b79c90e6e2a",
+  overlay: "3cf78c3623442ad827eed58a1780784a6eb95676",
 
   // ======================================================================================================
   // 
@@ -251,6 +251,8 @@
     // unittests 
     testGate(type="unittest", platform="linux"),
     testGate(type="unittest", platform="darwin"),
+    testGate(type="svm-unittest", platform="linux"),
+    testGate(type="svm-unittest", platform="darwin"),
 
     // junit 
     testGate(type="junit", platform="linux"),

graalpython/com.oracle.graal.python.cext/src/dictobject.c

@@ -57,6 +57,10 @@ PyObject* PyDict_GetItem(PyObject* d, PyObject* k) {
     return UPCALL_CEXT_O(_jls_PyDict_GetItem, native_to_java(d), native_to_java(k));
 }
 
+PyObject* _PyDict_GetItemId(PyObject* d, _Py_Identifier* id) {
+    return PyDict_GetItemString(d, id->string);
+}
+
 UPCALL_ID(PyDict_DelItem);
 int PyDict_DelItem(PyObject *d, PyObject *k) {
     return UPCALL_CEXT_I(_jls_PyDict_DelItem, native_to_java(d), native_to_java(k));

graalpython/com.oracle.graal.python.cext/src/exceptions.c

@@ -81,6 +81,7 @@ PyObject * PyExc_AssertionError = NULL;
 PyObject * PyExc_UnboundLocalError = NULL;
 PyObject * PyExc_NotImplementedError = NULL;
 PyObject * PyExc_RecursionError = NULL;
+PyObject * PyExc_UnicodeEncodeError = NULL;
 
 void initialize_exceptions() {
     PyExc_AttributeError = PY_EXCEPTION("AttributeError");
@@ -118,6 +119,7 @@ void initialize_exceptions() {
     PyExc_NotImplementedError = PY_EXCEPTION("NotImplementedError");
     PyExc_RecursionError = PY_EXCEPTION("RecursionError");
     PyExc_NotImplementedError = PY_EXCEPTION("NotImplementedError");
+    PyExc_UnicodeEncodeError = PY_EXCEPTION("UnicodeEncodeError");
 }
 
 

graalpython/com.oracle.graal.python.cext/src/object.c

@@ -388,6 +388,10 @@ PyObject* _PyObject_GC_New(PyTypeObject *tp) {
     return _PyObject_New(tp);
 }
 
+PyVarObject* _PyObject_GC_NewVar(PyTypeObject *tp, Py_ssize_t nitems) {
+    return _PyObject_NewVar(tp, nitems);
+}
+
 PyVarObject *
 _PyObject_NewVar(PyTypeObject *tp, Py_ssize_t nitems)
 {

graalpython/com.oracle.graal.python.shell/src/com/oracle/graal/python/shell/ConsoleHandler.java

@@ -109,4 +109,14 @@ public int read() throws IOException {
             }
         };
     }
+
+    public int getTerminalWidth() {
+        // deafult value
+        return 80;
+    }
+
+    public int getTerminalHeight() {
+        // default value
+        return 25;
+    }
 }

graalpython/com.oracle.graal.python.shell/src/com/oracle/graal/python/shell/GraalPythonCC.java

@@ -0,0 +1,266 @@
+/*
+ * Copyright (c) 2018, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * The Universal Permissive License (UPL), Version 1.0
+ *
+ * Subject to the condition set forth below, permission is hereby granted to any
+ * person obtaining a copy of this software, associated documentation and/or
+ * data (collectively the "Software"), free of charge and under any and all
+ * copyright rights in the Software, and any and all patent rights owned or
+ * freely licensable by each licensor hereunder covering either (i) the
+ * unmodified Software as contributed to or provided by such licensor, or (ii)
+ * the Larger Works (as defined below), to deal in both
+ *
+ * (a) the Software, and
+ *
+ * (b) any piece of software and/or hardware listed in the lrgrwrks.txt file if
+ * one is included with the Software each a "Larger Work" to which the Software
+ * is contributed by such licensors),
+ *
+ * without restriction, including without limitation the rights to copy, create
+ * derivative works of, display, perform, and distribute the Software and make,
+ * use, sell, offer for sale, import, export, have made, and have sold the
+ * Software and the Larger Work(s), and to sublicense the foregoing rights on
+ * either these or other terms.
+ *
+ * This license is subject to the following condition:
+ *
+ * The above copyright notice and either this complete permission notice or at a
+ * minimum a reference to the UPL must be included in all copies or substantial
+ * portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+package com.oracle.graal.python.shell;
+
+import java.io.IOException;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.Paths;
+import java.nio.file.StandardCopyOption;
+import java.nio.file.attribute.PosixFilePermission;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.HashSet;
+import java.util.List;
+
+public class GraalPythonCC extends GraalPythonCompiler {
+    private String outputFilename;
+    private boolean linkExecutable;
+    private boolean link;
+    private boolean linkLLI;
+    private Boolean compile;
+    private List<String> clangArgs;
+    private List<String> execLinkArgs;
+    private List<String> fileInputs;
+
+    GraalPythonCC() {
+    }
+
+    private static List<String> execLinkPrefix = Arrays.asList(new String[]{
+                    "clang",
+                    "-fembed-bitcode",
+                    "-fPIC",
+                    "-ggdb",
+                    "-O1",
+    });
+    private static List<String> clangPrefix = Arrays.asList(new String[]{
+                    "clang",
+                    "-emit-llvm",
+                    "-fPIC",
+                    "-Wno-int-to-void-pointer-cast",
+                    "-Wno-int-conversion",
+                    "-Wno-incompatible-pointer-types-discards-qualifiers",
+                    "-ggdb",
+                    "-O1",
+    });
+    private static List<String> optPrefix = Arrays.asList(new String[]{
+                    "opt",
+                    "-mem2reg",
+                    "-globalopt",
+                    "-simplifycfg",
+                    "-constprop",
+                    "-always-inline",
+                    "-instcombine",
+                    "-dse",
+                    "-loop-simplify",
+                    "-reassociate",
+                    "-licm",
+                    "-gvn",
+                    "-o",
+    });
+
+    static void main(String[] args) {
+        new GraalPythonCC().run(args);
+    }
+
+    private void run(String[] args) {
+        parseOptions(args);
+        launchCC();
+    }
+
+    private void parseOptions(String[] args) {
+        outputFilename = A_OUT;
+        linkExecutable = true;
+        linkLLI = false;
+        link = true;
+        verbose = false;
+        compile = null;
+        clangArgs = new ArrayList<>(clangPrefix);
+        execLinkArgs = new ArrayList<>(execLinkPrefix);
+        fileInputs = new ArrayList<>();
+        for (int i = 0; i < args.length; i++) {
+            String arg = args[i];
+            switch (arg) {
+                case "-o":
+                    clangArgs.add(arg);
+                    i++;
+                    if (i >= args.length) {
+                        throw new RuntimeException("-o needs an argument");
+                    }
+                    outputFilename = arg = args[i];
+                    break;
+                case "-shared":
+                    linkExecutable = false;
+                    break;
+                case "-c":
+                    link = false;
+                    linkExecutable = false;
+                    break;
+                case "--link-lli-scripts":
+                    linkLLI = true;
+                    continue; // skip adding this to clang args
+                case "-v":
+                    if (!verbose) {
+                        verbose = true;
+                        continue; // the first verbose is not passed on to clang
+                    }
+                    break;
+                default:
+                    if (arg.endsWith(".o") || arg.endsWith(".bc")) {
+                        if (compile == null) {
+                            compile = false;
+                        } else if (compile != false) {
+                            throw new RuntimeException("cannot mix source and compiled sources");
+                        }
+                        fileInputs.add(arg);
+                    } else if (arg.endsWith(".c") || arg.endsWith(".cpp") || arg.endsWith(".cxx")) {
+                        if (compile == null) {
+                            compile = true;
+                        } else if (compile != true) {
+                            throw new RuntimeException("cannot mix source and compiled sources");
+                        }
+                        fileInputs.add(arg);
+                    } else {
+                        execLinkArgs.add(arg);
+                    }
+            }
+            clangArgs.add(arg);
+        }
+        String targetFlags = System.getenv("LLVM_TARGET_FLAGS");
+        if (targetFlags != null) {
+            clangArgs.addAll(Arrays.asList(targetFlags.split(" ")));
+        }
+    }
+
+    private void launchCC() {
+        // run the clang compiler to generate bc files
+        try {
+            Files.delete(Paths.get(outputFilename));
+        } catch (IOException e) {
+            // no matter;
+        }
+
+        if (compile) {
+            exec(clangArgs);
+            logV("opt: ", fileInputs);
+            if (!Files.exists(Paths.get(outputFilename))) {
+                // if no explicit output filename was given or produced, we search the commandline
+                // for files for which now have a bc file and optimize those. This happens when you
+                // pass multiple .c files to clang and ask it to emit llvm bitcode
+                for (String f : fileInputs) {
+                    String bcFile = bcFileFromFilename(f);
+                    assert Files.exists(Paths.get(bcFile));
+                    optFile(bcFile);
+                    try {
+                        String objFile = objectFileFromFilename(f);
+                        logV("Optimized:", bcFile, "->", objFile);
+                        Files.move(Paths.get(bcFile), Paths.get(objFile));
+                    } catch (IOException e) {
+                        throw new RuntimeException(e);
+                    }
+                }
+            } else {
+                optFile(outputFilename);
+            }
+        }
+
+        if (link) {
+            linkShared(fileInputs);
+            if (linkExecutable) {
+                try {
+                    Path linkedBCfile = Files.move(Paths.get(outputFilename), Paths.get(bcFileFromFilename(outputFilename)), StandardCopyOption.REPLACE_EXISTING);
+                    linkExecutable(outputFilename, linkedBCfile.toString());
+                } catch (IOException e) {
+                    throw new RuntimeException(e);
+                }
+            }
+        }
+    }
+
+    private void linkExecutable(String executableScript, String linkedBcFile) throws IOException {
+        if (linkLLI) {
+            List<String> cmdline = GraalPythonMain.getCmdline(Arrays.asList(), Arrays.asList());
+            cmdline.add("-LLI");
+            cmdline.add(linkedBcFile);
+            Path executablePath = Paths.get(executableScript);
+            Files.write(executablePath, String.join(" ", cmdline).getBytes());
+            HashSet<PosixFilePermission> perms = new HashSet<>(Arrays.asList(new PosixFilePermission[]{PosixFilePermission.OWNER_EXECUTE, PosixFilePermission.GROUP_EXECUTE}));
+            perms.addAll(Files.getPosixFilePermissions(executablePath));
+        } else {
+            execLinkArgs.add(linkedBcFile);
+            execLinkArgs.add("-o");
+            execLinkArgs.add(executableScript);
+            exec(execLinkArgs);
+        }
+    }
+
+    private void linkShared(List<String> bitcodeFiles) {
+        if (fileInputs.size() > 0) {
+            ArrayList<String> ldCmd = new ArrayList<>(bitcodeFiles);
+            if (verbose) {
+                ldCmd.add("-v");
+            }
+            ldCmd.add("-o");
+            ldCmd.add(outputFilename);
+            GraalPythonLD.main(ldCmd.toArray(new String[0]));
+        }
+    }
+
+    private void optFile(String bcFile) {
+        List<String> opt = new ArrayList<>(optPrefix);
+        opt.add(bcFile);
+        opt.add(bcFile);
+        exec(opt);
+    }
+
+    private static String bcFileFromFilename(String f) {
+        int dotIdx = f.lastIndexOf('.');
+        if (dotIdx > 1) {
+            return f.substring(0, dotIdx + 1) + "bc";
+        } else {
+            return f + ".bc";
+        }
+    }
+
+    private static String objectFileFromFilename(String f) {
+        return bcFileFromFilename(f).replaceAll("\\.bc$", ".o");
+    }
+}