merge default into releaserelease-pypy2.7-v7.1.1

11 files changed, 132 insertions, 31 deletions

diff --git a/LICENSE b/LICENSE
index c7ca4d3e27..6b4336b88b 100644
--- a/LICENSE
+++ b/LICENSE
@@ -110,6 +110,7 @@ copyrighted by one or more of the following people and organizations:
   Devin Jeanpierre
   Bob Ippolito
   Bruno Gola
+  Andrew Lawrence
   David Malcolm
   Squeaky
   Edd Barrett
@@ -125,7 +126,6 @@ copyrighted by one or more of the following people and organizations:
   John Witulski
   Stefan Beyer
   Jeremy Thurgood
-  Andrew Lawrence
   Greg Price
   Ivan Sichmann Freitas
   Dario Bertini
@@ -254,6 +254,7 @@ copyrighted by one or more of the following people and organizations:
   Omer Katz
   Jacek Generowicz
   Tomasz Dziopa
+  Lin Cheng
   Sylvain Thenault
   Jakub Stasiak
   Andrew Dalke
@@ -403,6 +404,7 @@ copyrighted by one or more of the following people and organizations:
   Niclas Olofsson
   Chris Pressey
   Tobias Diaz
+  Paul Graydon
   Nikolaos-Digenis Karagiannis
   Kurt Griffiths
   Ben Mather
diff --git a/lib_pypy/cffi.egg-info/PKG-INFO b/lib_pypy/cffi.egg-info/PKG-INFO
index a2c12a1fc9..c223f85359 100644
--- a/lib_pypy/cffi.egg-info/PKG-INFO
+++ b/lib_pypy/cffi.egg-info/PKG-INFO
@@ -1,6 +1,6 @@
 Metadata-Version: 1.1
 Name: cffi
-Version: 1.12.2
+Version: 1.12.3
 Summary: Foreign Function Interface for Python calling C code.
 Home-page: http://cffi.readthedocs.org
 Author: Armin Rigo, Maciej Fijalkowski
diff --git a/lib_pypy/cffi/__init__.py b/lib_pypy/cffi/__init__.py
index 5ebb64b072..0224a15a7d 100644
--- a/lib_pypy/cffi/__init__.py
+++ b/lib_pypy/cffi/__init__.py
@@ -5,8 +5,8 @@ from .api import FFI
 from .error import CDefError, FFIError, VerificationError, VerificationMissing
 from .error import PkgConfigError
 
-__version__ = "1.12.2"
-__version_info__ = (1, 12, 2)
+__version__ = "1.12.3"
+__version_info__ = (1, 12, 3)
 
 # The verifier module file names are based on the CRC32 of a string that
 # contains the following version number.  It may be older than __version__
diff --git a/lib_pypy/cffi/_embedding.h b/lib_pypy/cffi/_embedding.h
index 09a784875c..fb851bbd9e 100644
--- a/lib_pypy/cffi/_embedding.h
+++ b/lib_pypy/cffi/_embedding.h
@@ -223,7 +223,7 @@ static int _cffi_initialize_python(void)
 
         if (f != NULL && f != Py_None) {
             PyFile_WriteString("\nFrom: " _CFFI_MODULE_NAME
-                               "\ncompiled with cffi version: 1.12.2"
+                               "\ncompiled with cffi version: 1.12.3"
                                "\n_cffi_backend module: ", f);
             modules = PyImport_GetModuleDict();
             mod = PyDict_GetItemString(modules, "_cffi_backend");
diff --git a/pypy/doc/contributor.rst b/pypy/doc/contributor.rst
index f2ebcdc7be..aba36105dd 100644
--- a/pypy/doc/contributor.rst
+++ b/pypy/doc/contributor.rst
@@ -77,6 +77,7 @@ Contributors
   Devin Jeanpierre
   Bob Ippolito
   Bruno Gola
+  Andrew Lawrence
   David Malcolm
   Squeaky
   Edd Barrett
@@ -92,7 +93,6 @@ Contributors
   John Witulski
   Stefan Beyer
   Jeremy Thurgood
-  Andrew Lawrence
   Greg Price
   Ivan Sichmann Freitas
   Dario Bertini
@@ -221,6 +221,7 @@ Contributors
   Omer Katz
   Jacek Generowicz
   Tomasz Dziopa
+  Lin Cheng
   Sylvain Thenault
   Jakub Stasiak
   Andrew Dalke
@@ -370,6 +371,7 @@ Contributors
   Niclas Olofsson
   Chris Pressey
   Tobias Diaz
+  Paul Graydon
   Nikolaos-Digenis Karagiannis
   Kurt Griffiths
   Ben Mather
diff --git a/pypy/doc/release-v7.1.1.rst b/pypy/doc/release-v7.1.1.rst
new file mode 100644
index 0000000000..b93ca84908
--- /dev/null
+++ b/pypy/doc/release-v7.1.1.rst
@@ -0,0 +1,98 @@
+=========================================
+PyPy v7.1.1: release of 2.7, and 3.6-beta
+=========================================
+
+The PyPy team is proud to release a bug-fix release version 7.1.1 of PyPy, which
+includes two different interpreters:
+
+  - PyPy2.7, which is an interpreter supporting the syntax and the features of
+    Python 2.7
+
+  - PyPy3.6-beta: this is the second official release of PyPy to support 3.6
+    features, although it is still considered beta quality.
+    
+The interpreters are based on much the same codebase, thus the double
+release.
+
+As always, this release is 100% compatible with the previous one and fixed
+several issues and bugs raised by the growing community of PyPy users.
+We strongly recommend updating.
+
+The PyPy3.6 release is still not production quality so your mileage may vary.
+There are open issues with incomplete compatibility and c-extension support.
+
+You can download the v7.1.1 releases here:
+
+    http://pypy.org/download.html
+
+We would like to thank our donors for the continued support of the PyPy
+project. If PyPy is not quite good enough for your needs, we are available for
+direct consulting work.
+
+We would also like to thank our contributors and encourage new people to join
+the project. PyPy has many layers and we need help with all of them: `PyPy`_
+and `RPython`_ documentation improvements, tweaking popular modules to run
+on pypy, or general `help`_ with making RPython's JIT even better.
+
+.. _`PyPy`: index.html
+.. _`RPython`: https://rpython.readthedocs.org
+.. _`help`: project-ideas.html
+
+What is PyPy?
+=============
+
+PyPy is a very compliant Python interpreter, almost a drop-in replacement for
+CPython 2.7, 3.6. It's fast (`PyPy and CPython 2.7.x`_ performance
+comparison) due to its integrated tracing JIT compiler.
+
+We also welcome developers of other `dynamic languages`_ to see what RPython
+can do for them.
+
+This PyPy release supports:
+
+  * **x86** machines on most common operating systems
+    (Linux 32/64 bits, Mac OS X 64 bits, Windows 32 bits, OpenBSD, FreeBSD)
+
+  * big- and little-endian variants of **PPC64** running Linux,
+
+  * **s390x** running Linux
+
+Unfortunately at the moment of writing our ARM buildbots are out of service,
+so for now we are **not** releasing any binary for the ARM architecture,
+although PyPy does support ARM 32 bit processors.
+
+.. _`PyPy and CPython 2.7.x`: http://speed.pypy.org
+.. _`dynamic languages`: http://rpython.readthedocs.io/en/latest/examples.html
+
+
+Changelog
+=========
+
+Changes shared across versions
+* improve performance of ``u''.append``
+* Prevent a crash in ``zlib`` when flushing a closed stream
+* Fix a few corener cases when encountering unicode values above 0x110000
+* Teach the JIT how to handle very large constant lists, sets, or dicts
+* Fix building on ARM32 (issue 2984_)
+* Fix a bug in register assignment in ARM32
+* Package windows DLLs needed by cffi modules next to the cffi c-extensions
+  (issue 2988_)
+* Cleanup and refactor JIT code to remove ``rpython.jit.metainterp.typesystem``
+* Fix memoryviews of ctype structures with padding, (cpython issue 32780_)
+* CFFI updated to as-yet-unreleased 1.12.3
+
+Python 3.6 only
+
+* On win32, override some ``errno.E*`` values that were added to MSVC in v2010
+  so that ``errno.E* == errno.WSAE*`` as in CPython
+* Do the same optimization that CPython does for ``(1, 2, 3, *a)`` (but at the
+  AST level)
+* ``str.maketrans`` was broken (issue 2991_)
+* Raise a ``TypeError`` when using buffers and unicode such as ``''.strip(buffer)``
+  and ``'a' < buffer``
+* Support ``_overlapped`` and asyncio on win32
+
+.. _2984: https://bitbucket.org/pypy/pypy/issues/2984
+.. _2991: https://bitbucket.org/pypy/pypy/issues/2991
+.. _2988: https://bitbucket.org/pypy/pypy/issues/2988
+.. _32780: https://bugs.python.org/issue32780
diff --git a/pypy/module/_cffi_backend/__init__.py b/pypy/module/_cffi_backend/__init__.py
index 3bc10277de..e1324772df 100644
--- a/pypy/module/_cffi_backend/__init__.py
+++ b/pypy/module/_cffi_backend/__init__.py
@@ -3,7 +3,7 @@ from pypy.interpreter.mixedmodule import MixedModule
 from rpython.rlib import rdynload, clibffi
 from rpython.rtyper.lltypesystem import rffi
 
-VERSION = "1.12.2"
+VERSION = "1.12.3"
 
 FFI_DEFAULT_ABI = clibffi.FFI_DEFAULT_ABI
 try:
diff --git a/pypy/module/_cffi_backend/test/_backend_test_c.py b/pypy/module/_cffi_backend/test/_backend_test_c.py
index 0b82017ade..5b5eea4021 100644
--- a/pypy/module/_cffi_backend/test/_backend_test_c.py
+++ b/pypy/module/_cffi_backend/test/_backend_test_c.py
@@ -1,7 +1,7 @@
 # ____________________________________________________________
 
 import sys
-assert __version__ == "1.12.2", ("This test_c.py file is for testing a version"
+assert __version__ == "1.12.3", ("This test_c.py file is for testing a version"
                                  " of cffi that differs from the one that we"
                                  " get from 'import _cffi_backend'")
 if sys.version_info < (3,):
diff --git a/pypy/module/array/interp_array.py b/pypy/module/array/interp_array.py
index 59e66e2b60..26a583953b 100644
--- a/pypy/module/array/interp_array.py
+++ b/pypy/module/array/interp_array.py
@@ -1053,21 +1053,17 @@ def make_array(mytype):
                 code = r_uint(ord(item))
                 # cpython will allow values > sys.maxunicode
                 # while silently truncating the top bits
-                if code <= r_uint(0x7F):
-                    # Encode ASCII
-                    item = chr(code)
-                elif code <= r_uint(0x07FF):
-                    item = (chr((0xc0 | (code >> 6))) + 
-                            chr((0x80 | (code & 0x3f))))
-                elif code <= r_uint(0xFFFF):
-                    item = (chr((0xe0 | (code >> 12))) +
-                            chr((0x80 | ((code >> 6) & 0x3f))) +
-                            chr((0x80 | (code & 0x3f))))
-                else:
-                    item = (chr((0xf0 | (code >> 18)) & 0xff) +
-                            chr((0x80 | ((code >> 12) & 0x3f))) +
-                            chr((0x80 | ((code >> 6) & 0x3f))) +
-                            chr((0x80 | (code & 0x3f))))
+                # For now I (arigo) am going to ignore that and
+                # raise a ValueError always here, instead of getting
+                # some invalid utf8-encoded string which makes things
+                # potentially explode left and right.
+                try:
+                    item = rutf8.unichr_as_utf8(code)
+                except rutf8.OutOfRange:
+                    raise oefmt(space.w_ValueError,
+                        "cannot operate on this array('u') because it contains"
+                        " character %s not in range [U+0000; U+10ffff]"
+                        " at index %d", 'U+%x' % code, idx)
                 return space.newutf8(item, 1)
             assert 0, "unreachable"
 
diff --git a/pypy/module/array/test/test_array.py b/pypy/module/array/test/test_array.py
index f370dc2128..7a3050979c 100644
--- a/pypy/module/array/test/test_array.py
+++ b/pypy/module/array/test/test_array.py
@@ -851,7 +851,13 @@ class AppTestArray(object):
         a = self.array('u', input_unicode)
         b = self.array('u', input_unicode)
         b.byteswap()
-        assert a != b
+        assert b[2] == u'\u0000'
+        raises(ValueError, "b[1]")        # doesn't work
+        e = raises(ValueError, "a != b")  # doesn't work
+        assert str(e.value) == (
+            "cannot operate on this array('u') because it contains"
+            " character U+1000000 not in range [U+0000; U+10ffff]"
+            " at index 0")
         assert str(a) == "array('u', %r)" % (input_unicode,)
         assert str(b) == ("array('u', <character U+1000000 is not in"
                           " range [U+0000; U+10ffff]>)")
diff --git a/pypy/objspace/std/unicodeobject.py b/pypy/objspace/std/unicodeobject.py
index b9a212183a..a89047cd16 100644
--- a/pypy/objspace/std/unicodeobject.py
+++ b/pypy/objspace/std/unicodeobject.py
@@ -42,13 +42,10 @@ class W_UnicodeObject(W_Root):
         self._length = length
         self._index_storage = rutf8.null_storage()
         if not we_are_translated():
-            try:
-                # best effort, too expensive to handle surrogates
-                ulength = rutf8.codepoints_in_utf(utf8str)
-            except:
-                ulength = length 
-            assert ulength == length
-
+            # utf8str must always be a valid utf8 string, except maybe with
+            # explicit surrogate characters---which .decode('utf-8') doesn't
+            # special-case in Python 2, which is exactly what we want here
+            assert length == len(utf8str.decode('utf-8'))
 
 
     @staticmethod