diff options
124 files changed, 2264 insertions, 1109 deletions
@@ -28,7 +28,7 @@ the 'rpython', 'pypy', 'ctype_configure', 'dotviewer', 'demo', 'lib_pypy', DEALINGS IN THE SOFTWARE. -PyPy Copyright holders 2003-2014 +PyPy Copyright holders 2003-2015 ----------------------------------- Except when otherwise stated (look for LICENSE files or information at diff --git a/lib-python/2.7/distutils/unixccompiler.py b/lib-python/2.7/distutils/unixccompiler.py index b55bfd7679..e7fcda9d5a 100644 --- a/lib-python/2.7/distutils/unixccompiler.py +++ b/lib-python/2.7/distutils/unixccompiler.py @@ -58,7 +58,7 @@ class UnixCCompiler(CCompiler): executables = {'preprocessor' : None, 'compiler' : ["cc"], 'compiler_so' : ["cc"], - 'compiler_cxx' : ["cc"], + 'compiler_cxx' : ["c++"], # pypy: changed, 'cc' is bogus 'linker_so' : ["cc", "-shared"], 'linker_exe' : ["cc"], 'archiver' : ["ar", "-cr"], diff --git a/lib-python/2.7/subprocess.py b/lib-python/2.7/subprocess.py index b14068a374..e6ab29c3d6 100644 --- a/lib-python/2.7/subprocess.py +++ b/lib-python/2.7/subprocess.py @@ -1589,7 +1589,7 @@ def _pypy_install_libs_after_virtualenv(target_executable): 'copyfile' in caller.f_globals): dest_dir = sys.pypy_resolvedirof(target_executable) src_dir = sys.pypy_resolvedirof(sys.executable) - for libname in ['libpypy-c.so']: + for libname in ['libpypy-c.so', 'libpypy-c.dylib']: dest_library = os.path.join(dest_dir, libname) src_library = os.path.join(src_dir, libname) if os.path.exists(src_library): diff --git a/lib-python/2.7/test/test_xml_etree.py b/lib-python/2.7/test/test_xml_etree.py index bb57468e11..8ba0e2c600 100644 --- a/lib-python/2.7/test/test_xml_etree.py +++ b/lib-python/2.7/test/test_xml_etree.py @@ -225,9 +225,9 @@ def simpleops(): >>> element.remove(subelement) >>> serialize(element) # 5 '<tag key="value" />' - >>> element.remove(subelement) + >>> element.remove(subelement) # doctest: +ELLIPSIS Traceback (most recent call last): - ValueError: list.remove(x): x not in list + ValueError: list.remove(... >>> serialize(element) # 6 '<tag key="value" />' >>> element[0:0] = [subelement, subelement, subelement] diff --git a/lib_pypy/_functools.py b/lib_pypy/_functools.py index 50c5e01d93..ebd5e3fd8d 100644 --- a/lib_pypy/_functools.py +++ b/lib_pypy/_functools.py @@ -9,7 +9,10 @@ class partial(object): of the given arguments and keywords. """ - def __init__(self, func, *args, **keywords): + def __init__(self, *args, **keywords): + if not args: + raise TypeError('__init__() takes at least 2 arguments (1 given)') + func, args = args[0], args[1:] if not callable(func): raise TypeError("the first argument must be callable") self._func = func diff --git a/lib_pypy/cffi/__init__.py b/lib_pypy/cffi/__init__.py index df981fd5f9..0fe44447c3 100644 --- a/lib_pypy/cffi/__init__.py +++ b/lib_pypy/cffi/__init__.py @@ -6,3 +6,8 @@ from .ffiplatform import VerificationError, VerificationMissing __version__ = "0.8.6" __version_info__ = (0, 8, 6) + +# The verifier module file names are based on the CRC32 of a string that +# contains the following version number. It may be older than __version__ +# if nothing is clearly incompatible. +__version_verifier_modules__ = "0.8.6" diff --git a/lib_pypy/cffi/api.py b/lib_pypy/cffi/api.py index aed971562b..2bc4da8a8e 100644 --- a/lib_pypy/cffi/api.py +++ b/lib_pypy/cffi/api.py @@ -69,6 +69,7 @@ class FFI(object): self._function_caches = [] self._libraries = [] self._cdefsources = [] + self._windows_unicode = None if hasattr(backend, 'set_ffi'): backend.set_ffi(self) for name in backend.__dict__: @@ -77,6 +78,7 @@ class FFI(object): # with self._lock: self.BVoidP = self._get_cached_btype(model.voidp_type) + self.BCharA = self._get_cached_btype(model.char_array_type) if isinstance(backend, types.ModuleType): # _cffi_backend: attach these constants to the class if not hasattr(FFI, 'NULL'): @@ -189,13 +191,16 @@ class FFI(object): cdecl = self._typeof(cdecl) return self._backend.alignof(cdecl) - def offsetof(self, cdecl, fieldname): + def offsetof(self, cdecl, *fields_or_indexes): """Return the offset of the named field inside the given - structure, which must be given as a C type name. + structure or array, which must be given as a C type name. + You can give several field names in case of nested structures. + You can also give numeric values which correspond to array + items, in case of an array type. """ if isinstance(cdecl, basestring): cdecl = self._typeof(cdecl) - return self._backend.typeoffsetof(cdecl, fieldname)[1] + return self._typeoffsetof(cdecl, *fields_or_indexes)[1] def new(self, cdecl, init=None): """Allocate an instance according to the specified C type and @@ -264,6 +269,16 @@ class FFI(object): """ return self._backend.buffer(cdata, size) + def from_buffer(self, python_buffer): + """Return a <cdata 'char[]'> that points to the data of the + given Python object, which must support the buffer interface. + Note that this is not meant to be used on the built-in types str, + unicode, or bytearray (you can build 'char[]' arrays explicitly) + but only on objects containing large quantities of raw data + in some other format, like 'array.array' or numpy arrays. + """ + return self._backend.from_buffer(self.BCharA, python_buffer) + def callback(self, cdecl, python_callable=None, error=None): """Return a callback object or a decorator making such a callback object. 'cdecl' must name a C function pointer type. @@ -335,9 +350,23 @@ class FFI(object): which requires binary compatibility in the signatures. """ from .verifier import Verifier, _caller_dir_pycache + # + # If set_unicode(True) was called, insert the UNICODE and + # _UNICODE macro declarations + if self._windows_unicode: + self._apply_windows_unicode(kwargs) + # + # Set the tmpdir here, and not in Verifier.__init__: it picks + # up the caller's directory, which we want to be the caller of + # ffi.verify(), as opposed to the caller of Veritier(). tmpdir = tmpdir or _caller_dir_pycache() + # + # Make a Verifier() and use it to load the library. self.verifier = Verifier(self, source, tmpdir, **kwargs) lib = self.verifier.load_library() + # + # Save the loaded library for keep-alive purposes, even + # if the caller doesn't keep it alive itself (it should). self._libraries.append(lib) return lib @@ -356,15 +385,29 @@ class FFI(object): with self._lock: return model.pointer_cache(self, ctype) - def addressof(self, cdata, field=None): + def addressof(self, cdata, *fields_or_indexes): """Return the address of a <cdata 'struct-or-union'>. - If 'field' is specified, return the address of this field. + If 'fields_or_indexes' are given, returns the address of that + field or array item in the structure or array, recursively in + case of nested structures. """ ctype = self._backend.typeof(cdata) - ctype, offset = self._backend.typeoffsetof(ctype, field) + if fields_or_indexes: + ctype, offset = self._typeoffsetof(ctype, *fields_or_indexes) + else: + if ctype.kind == "pointer": + raise TypeError("addressof(pointer)") + offset = 0 ctypeptr = self._pointer_to(ctype) return self._backend.rawaddressof(ctypeptr, cdata, offset) + def _typeoffsetof(self, ctype, field_or_index, *fields_or_indexes): + ctype, offset = self._backend.typeoffsetof(ctype, field_or_index) + for field1 in fields_or_indexes: + ctype, offset1 = self._backend.typeoffsetof(ctype, field1, 1) + offset += offset1 + return ctype, offset + def include(self, ffi_to_include): """Includes the typedefs, structs, unions and enums defined in another FFI instance. Usage is similar to a #include in C, @@ -387,6 +430,44 @@ class FFI(object): def from_handle(self, x): return self._backend.from_handle(x) + def set_unicode(self, enabled_flag): + """Windows: if 'enabled_flag' is True, enable the UNICODE and + _UNICODE defines in C, and declare the types like TCHAR and LPTCSTR + to be (pointers to) wchar_t. If 'enabled_flag' is False, + declare these types to be (pointers to) plain 8-bit characters. + This is mostly for backward compatibility; you usually want True. + """ + if self._windows_unicode is not None: + raise ValueError("set_unicode() can only be called once") + enabled_flag = bool(enabled_flag) + if enabled_flag: + self.cdef("typedef wchar_t TBYTE;" + "typedef wchar_t TCHAR;" + "typedef const wchar_t *LPCTSTR;" + "typedef const wchar_t *PCTSTR;" + "typedef wchar_t *LPTSTR;" + "typedef wchar_t *PTSTR;" + "typedef TBYTE *PTBYTE;" + "typedef TCHAR *PTCHAR;") + else: + self.cdef("typedef char TBYTE;" + "typedef char TCHAR;" + "typedef const char *LPCTSTR;" + "typedef const char *PCTSTR;" + "typedef char *LPTSTR;" + "typedef char *PTSTR;" + "typedef TBYTE *PTBYTE;" + "typedef TCHAR *PTCHAR;") + self._windows_unicode = enabled_flag + + def _apply_windows_unicode(self, kwds): + defmacros = kwds.get('define_macros', ()) + if not isinstance(defmacros, (list, tuple)): + raise TypeError("'define_macros' must be a list or tuple") + defmacros = list(defmacros) + [('UNICODE', '1'), + ('_UNICODE', '1')] + kwds['define_macros'] = defmacros + def _load_backend_lib(backend, name, flags): if name is None: diff --git a/lib_pypy/cffi/backend_ctypes.py b/lib_pypy/cffi/backend_ctypes.py index 2b2b481943..7fcde57fbc 100644 --- a/lib_pypy/cffi/backend_ctypes.py +++ b/lib_pypy/cffi/backend_ctypes.py @@ -169,6 +169,7 @@ class CTypesGenericArray(CTypesData): class CTypesGenericPtr(CTypesData): __slots__ = ['_address', '_as_ctype_ptr'] _automatic_casts = False + kind = "pointer" @classmethod def _newp(cls, init): @@ -370,10 +371,12 @@ class CTypesBackend(object): (CTypesPrimitive, type(source).__name__)) return source # + kind1 = kind class CTypesPrimitive(CTypesGenericPrimitive): __slots__ = ['_value'] _ctype = ctype _reftypename = '%s &' % name + kind = kind1 def __init__(self, value): self._value = value @@ -703,12 +706,13 @@ class CTypesBackend(object): class struct_or_union(base_ctypes_class): pass struct_or_union.__name__ = '%s_%s' % (kind, name) + kind1 = kind # class CTypesStructOrUnion(CTypesBaseStructOrUnion): __slots__ = ['_blob'] _ctype = struct_or_union _reftypename = '%s &' % (name,) - _kind = kind + _kind = kind = kind1 # CTypesStructOrUnion._fix_class() return CTypesStructOrUnion @@ -994,27 +998,42 @@ class CTypesBackend(object): def getcname(self, BType, replace_with): return BType._get_c_name(replace_with) - def typeoffsetof(self, BType, fieldname): - if fieldname is not None and issubclass(BType, CTypesGenericPtr): - BType = BType._BItem - if not issubclass(BType, CTypesBaseStructOrUnion): - raise TypeError("expected a struct or union ctype") - if fieldname is None: - return (BType, 0) - else: + def typeoffsetof(self, BType, fieldname, num=0): + if isinstance(fieldname, str): + if num == 0 and issubclass(BType, CTypesGenericPtr): + BType = BType._BItem + if not issubclass(BType, CTypesBaseStructOrUnion): + raise TypeError("expected a struct or union ctype") BField = BType._bfield_types[fieldname] if BField is Ellipsis: raise TypeError("not supported for bitfields") return (BField, BType._offsetof(fieldname)) + elif isinstance(fieldname, (int, long)): + if issubclass(BType, CTypesGenericArray): + BType = BType._CTPtr + if not issubclass(BType, CTypesGenericPtr): + raise TypeError("expected an array or ptr ctype") + BItem = BType._BItem + offset = BItem._get_size() * fieldname + if offset > sys.maxsize: + raise OverflowError + return (BItem, offset) + else: + raise TypeError(type(fieldname)) - def rawaddressof(self, BTypePtr, cdata, offset): + def rawaddressof(self, BTypePtr, cdata, offset=None): if isinstance(cdata, CTypesBaseStructOrUnion): ptr = ctypes.pointer(type(cdata)._to_ctypes(cdata)) elif isinstance(cdata, CTypesGenericPtr): + if offset is None or not issubclass(type(cdata)._BItem, + CTypesBaseStructOrUnion): + raise TypeError("unexpected cdata type") + ptr = type(cdata)._to_ctypes(cdata) + elif isinstance(cdata, CTypesGenericArray): ptr = type(cdata)._to_ctypes(cdata) else: raise TypeError("expected a <cdata 'struct-or-union'>") - if offset != 0: + if offset: ptr = ctypes.cast( ctypes.c_void_p( ctypes.cast(ptr, ctypes.c_void_p).value + offset), diff --git a/lib_pypy/cffi/commontypes.py b/lib_pypy/cffi/commontypes.py index 9daf2ffe53..04a5868f20 100644 --- a/lib_pypy/cffi/commontypes.py +++ b/lib_pypy/cffi/commontypes.py @@ -29,6 +29,9 @@ def resolve_common_type(commontype): result = model.PointerType(resolve_common_type(result[:-2])) elif result in model.PrimitiveType.ALL_PRIMITIVE_TYPES: result = model.PrimitiveType(result) + elif result == 'set-unicode-needed': + raise api.FFIError("The Windows type %r is only available after " + "you call ffi.set_unicode()" % (commontype,)) else: if commontype == result: raise api.FFIError("Unsupported type: %r. Please file a bug " @@ -86,8 +89,6 @@ def win_common_types(maxsize): "ULONGLONG": "unsigned long long", "WCHAR": "wchar_t", "SHORT": "short", - "TBYTE": "WCHAR", - "TCHAR": "WCHAR", "UCHAR": "unsigned char", "UINT": "unsigned int", "UINT8": "unsigned char", @@ -157,14 +158,12 @@ def win_common_types(maxsize): "LPCVOID": model.const_voidp_type, "LPCWSTR": "const WCHAR *", - "LPCTSTR": "LPCWSTR", "LPDWORD": "DWORD *", "LPHANDLE": "HANDLE *", "LPINT": "int *", "LPLONG": "long *", "LPSTR": "CHAR *", "LPWSTR": "WCHAR *", - "LPTSTR": "LPWSTR", "LPVOID": model.voidp_type, "LPWORD": "WORD *", "LRESULT": "LONG_PTR", @@ -173,7 +172,6 @@ def win_common_types(maxsize): "PBYTE": "BYTE *", "PCHAR": "CHAR *", "PCSTR": "const CHAR *", - "PCTSTR": "LPCWSTR", "PCWSTR": "const WCHAR *", "PDWORD": "DWORD *", "PDWORDLONG": "DWORDLONG *", @@ -200,9 +198,6 @@ def win_common_types(maxsize): "PSIZE_T": "SIZE_T *", "PSSIZE_T": "SSIZE_T *", "PSTR": "CHAR *", - "PTBYTE": "TBYTE *", - "PTCHAR": "TCHAR *", - "PTSTR": "LPWSTR", "PUCHAR": "UCHAR *", "PUHALF_PTR": "UHALF_PTR *", "PUINT": "UINT *", @@ -240,6 +235,15 @@ def win_common_types(maxsize): "USN": "LONGLONG", "VOID": model.void_type, "WPARAM": "UINT_PTR", + + "TBYTE": "set-unicode-needed", + "TCHAR": "set-unicode-needed", + "LPCTSTR": "set-unicode-needed", + "PCTSTR": "set-unicode-needed", + "LPTSTR": "set-unicode-needed", + "PTSTR": "set-unicode-needed", + "PTBYTE": "set-unicode-needed", + "PTCHAR": "set-unicode-needed", }) return result diff --git a/lib_pypy/cffi/cparser.py b/lib_pypy/cffi/cparser.py index a53d4c3fc4..49ca9b9503 100644 --- a/lib_pypy/cffi/cparser.py +++ b/lib_pypy/cffi/cparser.py @@ -1,4 +1,3 @@ - from . import api, model from .commontypes import COMMON_TYPES, resolve_common_type try: @@ -209,6 +208,8 @@ class Parser(object): def _add_constants(self, key, val): if key in self._int_constants: + if self._int_constants[key] == val: + return # ignore identical double declarations raise api.FFIError( "multiple declarations of constant: %s" % (key,)) self._int_constants[key] = val @@ -228,12 +229,18 @@ class Parser(object): pyvalue = int(int_str, 0) self._add_constants(key, pyvalue) + self._declare('macro ' + key, pyvalue) elif value == '...': self._declare('macro ' + key, value) else: - raise api.CDefError('only supports the syntax "#define ' - '%s ..." (literally) or "#define ' - '%s 0x1FF" for now' % (key, key)) + raise api.CDefError( + 'only supports one of the following syntax:\n' + ' #define %s ... (literally dot-dot-dot)\n' + ' #define %s NUMBER (with NUMBER an integer' + ' constant, decimal/hex/octal)\n' + 'got:\n' + ' #define %s %s' + % (key, key, key, value)) def _parse_decl(self, decl): node = decl.type @@ -460,6 +467,8 @@ class Parser(object): elif kind == 'union': tp = model.UnionType(explicit_name, None, None, None) elif kind == 'enum': + if explicit_name == '__dotdotdot__': + raise CDefError("Enums cannot be declared with ...") tp = self._build_enum_type(explicit_name, type.values) else: raise AssertionError("kind = %r" % (kind,)) @@ -532,9 +541,24 @@ class Parser(object): def _parse_constant(self, exprnode, partial_length_ok=False): # for now, limited to expressions that are an immediate number - # or negative number + # or positive/negative number if isinstance(exprnode, pycparser.c_ast.Constant): - return int(exprnode.value, 0) + s = exprnode.value + if s.startswith('0'): + if s.startswith('0x') or s.startswith('0X'): + return int(s, 16) + return int(s, 8) + elif '1' <= s[0] <= '9': + return int(s, 10) + elif s[0] == "'" and s[-1] == "'" and ( + len(s) == 3 or (len(s) == 4 and s[1] == "\\")): + return ord(s[-2]) + else: + raise api.CDefError("invalid constant %r" % (s,)) + # + if (isinstance(exprnode, pycparser.c_ast.UnaryOp) and + exprnode.op == '+'): + return self._parse_constant(exprnode.expr) # if (isinstance(exprnode, pycparser.c_ast.UnaryOp) and exprnode.op == '-'): diff --git a/lib_pypy/cffi/ffiplatform.py b/lib_pypy/cffi/ffiplatform.py index 4515d6c7b5..365b3e8a60 100644 --- a/lib_pypy/cffi/ffiplatform.py +++ b/lib_pypy/cffi/ffiplatform.py @@ -11,6 +11,9 @@ class VerificationMissing(Exception): """ +LIST_OF_FILE_NAMES = ['sources', 'include_dirs', 'library_dirs', + 'extra_objects', 'depends'] + def get_extension(srcfilename, modname, sources=(), **kwds): from distutils.core import Extension allsources = [srcfilename] diff --git a/lib_pypy/cffi/model.py b/lib_pypy/cffi/model.py index 371153fecb..dc0fa4a681 100644 --- a/lib_pypy/cffi/model.py +++ b/lib_pypy/cffi/model.py @@ -235,6 +235,8 @@ class ArrayType(BaseType): BPtrItem = PointerType(self.item).get_cached_btype(ffi, finishlist) return global_cache(self, ffi, 'new_array_type', BPtrItem, self.length) +char_array_type = ArrayType(PrimitiveType('char'), None) + class StructOrUnionOrEnum(BaseTypeByIdentity): _attrs_ = ('name',) @@ -478,7 +480,7 @@ def global_cache(srctype, ffi, funcname, *args, **kwds): try: res = getattr(ffi._backend, funcname)(*args) except NotImplementedError as e: - raise NotImplementedError("%r: %s" % (srctype, e)) + raise NotImplementedError("%s: %r: %s" % (funcname, srctype, e)) # note that setdefault() on WeakValueDictionary is not atomic # and contains a rare bug (http://bugs.python.org/issue19542); # we have to use a lock and do it ourselves diff --git a/lib_pypy/cffi/vengine_cpy.py b/lib_pypy/cffi/vengine_cpy.py index 31793f0381..1c4668cd01 100644 --- a/lib_pypy/cffi/vengine_cpy.py +++ b/lib_pypy/cffi/vengine_cpy.py @@ -65,7 +65,7 @@ class VCPythonEngine(object): # The following two 'chained_list_constants' items contains # the head of these two chained lists, as a string that gives the # call to do, if any. - self._chained_list_constants = ['0', '0'] + self._chained_list_constants = ['((void)lib,0)', '((void)lib,0)'] # prnt = self._prnt # first paste some standard set of lines that are mostly '#define' @@ -138,15 +138,22 @@ class VCPythonEngine(object): prnt() prnt('#endif') - def load_library(self): + def load_library(self, flags=None): # XXX review all usages of 'self' here! # import it as a new extension module + if hasattr(sys, "getdlopenflags"): + previous_flags = sys.getdlopenflags() try: + if hasattr(sys, "setdlopenflags") and flags is not None: + sys.setdlopenflags(flags) module = imp.load_dynamic(self.verifier.get_module_name(), self.verifier.modulefilename) except ImportError as e: error = "importing %r: %s" % (self.verifier.modulefilename, e) raise ffiplatform.VerificationError(error) + finally: + if hasattr(sys, "setdlopenflags"): + sys.setdlopenflags(previous_flags) # # call loading_cpy_struct() to get the struct layout inferred by # the C compiler @@ -228,7 +235,8 @@ class VCPythonEngine(object): converter = '_cffi_to_c_int' extraarg = ', %s' % tp.name else: - converter = '_cffi_to_c_%s' % (tp.name.replace(' ', '_'),) + converter = '(%s)_cffi_to_c_%s' % (tp.get_c_name(''), + tp.name.replace(' ', '_')) errvalue = '-1' # elif isinstance(tp, model.PointerType): @@ -267,8 +275,8 @@ class VCPythonEngine(object): self._prnt(' if (datasize != 0) {') self._prnt(' if (datasize < 0)') self._prnt(' %s;' % errcode) - self._prnt(' %s = alloca(datasize);' % (tovar,)) - self._prnt(' memset((void *)%s, 0, datasize);' % (tovar,)) + self._prnt(' %s = alloca((size_t)datasize);' % (tovar,)) + self._prnt(' memset((void *)%s, 0, (size_t)datasize);' % (tovar,)) self._prnt(' if (_cffi_convert_array_from_object(' '(char *)%s, _cffi_type(%d), %s) < 0)' % ( tovar, self._gettypenum(tp), fromvar)) @@ -336,7 +344,7 @@ class VCPythonEngine(object): prnt = self._prnt numargs = len(tp.args) if numargs == 0: - argname = 'no_arg' + argname = 'noarg' elif numargs == 1: argname = 'arg0' else: @@ -386,6 +394,9 @@ class VCPythonEngine(object): prnt(' Py_END_ALLOW_THREADS') prnt() # + prnt(' (void)self; /* unused */') + if numargs == 0: + prnt(' (void)noarg; /* unused */') if result_code: prnt(' return %s;' % self._convert_expr_from_c(tp.result, 'result', 'result type')) @@ -452,6 +463,7 @@ class VCPythonEngine(object): prnt('static void %s(%s *p)' % (checkfuncname, cname)) prnt('{') prnt(' /* only to generate compile-time warnings or errors */') + prnt(' (void)p;') for fname, ftype, fbitsize in tp.enumfields(): if (isinstance(ftype, model.PrimitiveType) and ftype.is_integer_type()) or fbitsize >= 0: @@ -482,6 +494,8 @@ class VCPythonEngine(object): prnt(' sizeof(((%s *)0)->%s),' % (cname, fname)) prnt(' -1') prnt(' };') + prnt(' (void)self; /* unused */') + prnt(' (void)noarg; /* unused */') prnt(' return _cffi_get_struct_layout(nums);') prnt(' /* the next line is not executed, but compiled */') prnt(' %s(0);' % (checkfuncname,)) @@ -578,7 +592,8 @@ class VCPythonEngine(object): # constants, likely declared with '#define' def _generate_cpy_const(self, is_int, name, tp=None, category='const', - vartp=None, delayed=True, size_too=False): + vartp=None, delayed=True, size_too=False, + check_value=None): prnt = self._prnt funcname = '_cffi_%s_%s' % (category, name) prnt('static int %s(PyObject *lib)' % funcname) @@ -590,6 +605,9 @@ class VCPythonEngine(object): else: assert category == 'const' # + if check_value is not None: + self._check_int_constant_value(name, check_value) + # if not is_int: if category == 'var': realexpr = '&' + name @@ -637,6 +655,27 @@ class VCPythonEngine(object): # ---------- # enums + def _check_int_constant_value(self, name, value, err_prefix=''): + prnt = self._prnt + if value <= 0: + prnt(' if ((%s) > 0 || (long)(%s) != %dL) {' % ( + name, name, value)) + else: + prnt(' if ((%s) <= 0 || (unsigned long)(%s) != %dUL) {' % ( + name, name, value)) + prnt(' char buf[64];') + prnt(' if ((%s) <= 0)' % name) + prnt(' snprintf(buf, 63, "%%ld", (long)(%s));' % name) + prnt(' else') + prnt(' snprintf(buf, 63, "%%lu", (unsigned long)(%s));' % + name) + prnt(' PyErr_Format(_cffi_VerificationError,') + prnt(' "%s%s has the real value %s, not %s",') + prnt(' "%s", "%s", buf, "%d");' % ( + err_prefix, name, value)) + prnt(' return -1;') + prnt(' }') + def _enum_funcname(self, prefix, name): # "$enum_$1" => "___D_enum____D_1" name = name.replace('$', '___D_') @@ -653,25 +692,8 @@ class VCPythonEngine(object): prnt('static int %s(PyObject *lib)' % funcname) prnt('{') for enumerator, enumvalue in zip(tp.enumerators, tp.enumvalues): - if enumvalue < 0: - prnt(' if ((%s) >= 0 || (long)(%s) != %dL) {' % ( - enumerator, enumerator, enumvalue)) - else: - prnt(' if ((%s) < 0 || (unsigned long)(%s) != %dUL) {' % ( - enumerator, enumerator, enumvalue)) - prnt(' char buf[64];') - prnt(' if ((%s) < 0)' % enumerator) - prnt(' snprintf(buf, 63, "%%ld", (long)(%s));' % enumerator) - prnt(' else') - prnt(' snprintf(buf, 63, "%%lu", (unsigned long)(%s));' % - enumerator) - prnt(' PyErr_Format(_cffi_VerificationError,') - prnt(' "enum %s: %s has the real value %s, ' - 'not %s",') - prnt(' "%s", "%s", buf, "%d");' % ( - name, enumerator, enumvalue)) - prnt(' return -1;') - prnt(' }') + self._check_int_constant_value(enumerator, enumvalue, + "enum %s: " % name) prnt(' return %s;' % self._chained_list_constants[True]) self._chained_list_constants[True] = funcname + '(lib)' prnt('}') @@ -695,8 +717,11 @@ class VCPythonEngine(object): # macros: for now only for integers def _generate_cpy_macro_decl(self, tp, name): - assert tp == '...' - self._generate_cpy_const(True, name) + if tp == '...': + check_value = None + else: + check_value = tp # an integer + self._generate_cpy_const(True, name, check_value=check_value) _generate_cpy_macro_collecttype = _generate_nothing _generate_cpy_macro_method = _generate_nothing @@ -783,6 +808,24 @@ cffimod_header = r''' typedef unsigned __int16 uint16_t; typedef unsigned __int32 uint32_t; typedef unsigned __int64 uint64_t; + typedef __int8 int_least8_t; + typedef __int16 int_least16_t; + typedef __int32 int_least32_t; + typedef __int64 int_least64_t; + typedef unsigned __int8 uint_least8_t; + typedef unsigned __int16 uint_least16_t; + typedef unsigned __int32 uint_least32_t; + typedef unsigned __int64 uint_least64_t; + typedef __int8 int_fast8_t; + typedef __int16 int_fast16_t; + typedef __int32 int_fast32_t; + typedef __int64 int_fast64_t; + typedef unsigned __int8 uint_fast8_t; + typedef unsigned __int16 uint_fast16_t; + typedef unsigned __int32 uint_fast32_t; + typedef unsigned __int64 uint_fast64_t; + typedef __int64 intmax_t; + typedef unsigned __int64 uintmax_t; # else # include <stdint.h> # endif @@ -828,12 +871,15 @@ cffimod_header = r''' PyLong_FromLongLong((long long)(x))) #define _cffi_from_c_int(x, type) \ - (((type)-1) > 0 ? /* unsigned */ \ - (sizeof(type) < sizeof(long) ? PyInt_FromLong(x) : \ - sizeof(type) == sizeof(long) ? PyLong_FromUnsignedLong(x) : \ - PyLong_FromUnsignedLongLong(x)) \ - : (sizeof(type) <= sizeof(long) ? PyInt_FromLong(x) : \ - PyLong_FromLongLong(x))) + (((type)-1) > 0 ? /* unsigned */ \ + (sizeof(type) < sizeof(long) ? \ + PyInt_FromLong((long)x) : \ + sizeof(type) == sizeof(long) ? \ + PyLong_FromUnsignedLong((unsigned long)x) : \ + PyLong_FromUnsignedLongLong((unsigned long long)x)) : \ + (sizeof(type) <= sizeof(long) ? \ + PyInt_FromLong((long)x) : \ + PyLong_FromLongLong((long long)x))) #define _cffi_to_c_int(o, type) \ (sizeof(type) == 1 ? (((type)-1) > 0 ? (type)_cffi_to_c_u8(o) \ @@ -844,7 +890,7 @@ cffimod_header = r''' : (type)_cffi_to_c_i32(o)) : \ sizeof(type) == 8 ? (((type)-1) > 0 ? (type)_cffi_to_c_u64(o) \ : (type)_cffi_to_c_i64(o)) : \ - (Py_FatalError("unsupported size for type " #type), 0)) + (Py_FatalError("unsupported size for type " #type), (type)0)) #define _cffi_to_c_i8 \ ((int(*)(PyObject *))_cffi_exports[1]) @@ -907,6 +953,7 @@ static PyObject *_cffi_setup(PyObject *self, PyObject *args) { PyObject *library; int was_alive = (_cffi_types != NULL); + (void)self; /* unused */ if (!PyArg_ParseTuple(args, "OOO", &_cffi_types, &_cffi_VerificationError, &library)) return NULL; diff --git a/lib_pypy/cffi/vengine_gen.py b/lib_pypy/cffi/vengine_gen.py index 133ec7f3fa..8e1412cf8c 100644 --- a/lib_pypy/cffi/vengine_gen.py +++ b/lib_pypy/cffi/vengine_gen.py @@ -58,12 +58,12 @@ class VGenericEngine(object): modname = self.verifier.get_module_name() prnt("void %s%s(void) { }\n" % (prefix, modname)) - def load_library(self): + def load_library(self, flags=0): # import it with the CFFI backend backend = self.ffi._backend # needs to make a path that contains '/', on Posix filename = os.path.join(os.curdir, self.verifier.modulefilename) - module = backend.load_library(filename) + module = backend.load_library(filename, flags) # # call loading_gen_struct() to get the struct layout inferred by # the C compiler @@ -235,6 +235,7 @@ class VGenericEngine(object): prnt('static void %s(%s *p)' % (checkfuncname, cname)) prnt('{') prnt(' /* only to generate compile-time warnings or errors */') + prnt(' (void)p;') for fname, ftype, fbitsize in tp.enumfields(): if (isinstance(ftype, model.PrimitiveType) and ftype.is_integer_type()) or fbitsize >= 0: @@ -354,11 +355,20 @@ class VGenericEngine(object): # ---------- # constants, likely declared with '#define' - def _generate_gen_const(self, is_int, name, tp=None, category='const'): + def _generate_gen_const(self, is_int, name, tp=None, category='const', + check_value=None): prnt = self._prnt funcname = '_cffi_%s_%s' % (category, name) self.export_symbols.append(funcname) - if is_int: + if check_value is not None: + assert is_int + assert category == 'const' + prnt('int %s(char *out_error)' % funcname) + prnt('{') + self._check_int_constant_value(name, check_value) + prnt(' return 0;') + prnt('}') + elif is_int: assert category == 'const' prnt('int %s(long long *out_value)' % funcname) prnt('{') @@ -367,6 +377,7 @@ class VGenericEngine(object): prnt('}') else: assert tp is not None + assert check_value is None prnt(tp.get_c_name(' %s(void)' % funcname, name),) prnt('{') if category == 'var': @@ -383,9 +394,13 @@ class VGenericEngine(object): _loading_gen_constant = _loaded_noop - def _load_constant(self, is_int, tp, name, module): + def _load_constant(self, is_int, tp, name, module, check_value=None): funcname = '_cffi_const_%s' % name - if is_int: + if check_value is not None: + assert is_int + self._load_known_int_constant(module, funcname) + value = check_value + elif is_int: BType = self.ffi._typeof_locked("long long*")[0] BFunc = self.ffi._typeof_locked("int(*)(long long*)")[0] function = module.load_function(BFunc, funcname) @@ -396,6 +411,7 @@ class VGenericEngine(object): BLongLong = self.ffi._typeof_locked("long long")[0] value += (1 << (8*self.ffi.sizeof(BLongLong))) else: + assert check_value is None BFunc = self.ffi._typeof_locked(tp.get_c_name('(*)(void)', name))[0] function = module.load_function(BFunc, funcname) value = function() @@ -410,6 +426,36 @@ class VGenericEngine(object): # ---------- # enums + def _check_int_constant_value(self, name, value): + prnt = self._prnt + if value <= 0: + prnt(' if ((%s) > 0 || (long)(%s) != %dL) {' % ( + name, name, value)) + else: + prnt(' if ((%s) <= 0 || (unsigned long)(%s) != %dUL) {' % ( + name, name, value)) + prnt(' char buf[64];') + prnt(' if ((%s) <= 0)' % name) + prnt(' sprintf(buf, "%%ld", (long)(%s));' % name) + prnt(' else') + prnt(' sprintf(buf, "%%lu", (unsigned long)(%s));' % + name) + prnt(' sprintf(out_error, "%s has the real value %s, not %s",') + prnt(' "%s", buf, "%d");' % (name[:100], value)) + prnt(' return -1;') + prnt(' }') + + def _load_known_int_constant(self, module, funcname): + BType = self.ffi._typeof_locked("char[]")[0] + BFunc = self.ffi._typeof_locked("int(*)(char*)")[0] + function = module.load_function(BFunc, funcname) + p = self.ffi.new(BType, 256) + if function(p) < 0: + error = self.ffi.string(p) + if sys.version_info >= (3,): + error = str(error, 'utf-8') + raise ffiplatform.VerificationError(error) + def _enum_funcname(self, prefix, name): # "$enum_$1" => "___D_enum____D_1" name = name.replace('$', '___D_') @@ -427,24 +473,7 @@ class VGenericEngine(object): prnt('int %s(char *out_error)' % funcname) prnt('{') for enumerator, enumvalue in zip(tp.enumerators, tp.enumvalues): - if enumvalue < 0: - prnt(' if ((%s) >= 0 || (long)(%s) != %dL) {' % ( - enumerator, enumerator, enumvalue)) - else: - prnt(' if ((%s) < 0 || (unsigned long)(%s) != %dUL) {' % ( - enumerator, enumerator, enumvalue)) - prnt(' char buf[64];') - prnt(' if ((%s) < 0)' % enumerator) - prnt(' sprintf(buf, "%%ld", (long)(%s));' % enumerator) - prnt(' else') - prnt(' sprintf(buf, "%%lu", (unsigned long)(%s));' % - enumerator) - prnt(' sprintf(out_error,' - ' "%s has the real value %s, not %s",') - prnt(' "%s", buf, "%d");' % ( - enumerator[:100], enumvalue)) - prnt(' return -1;') - prnt(' }') + self._check_int_constant_value(enumerator, enumvalue) prnt(' return 0;') prnt('}') prnt() @@ -456,16 +485,8 @@ class VGenericEngine(object): tp.enumvalues = tuple(enumvalues) tp.partial_resolved = True else: - BType = self.ffi._typeof_locked("char[]")[0] - BFunc = self.ffi._typeof_locked("int(*)(char*)")[0] funcname = self._enum_funcname(prefix, name) - function = module.load_function(BFunc, funcname) - p = self.ffi.new(BType, 256) - if function(p) < 0: - error = self.ffi.string(p) - if sys.version_info >= (3,): - error = str(error, 'utf-8') - raise ffiplatform.VerificationError(error) + self._load_known_int_constant(module, funcname) def _loaded_gen_enum(self, tp, name, module, library): for enumerator, enumvalue in zip(tp.enumerators, tp.enumvalues): @@ -476,13 +497,21 @@ class VGenericEngine(object): # macros: for now only for integers def _generate_gen_macro_decl(self, tp, name): - assert tp == '...' - self._generate_gen_const(True, name) + if tp == '...': + check_value = None + else: + check_value = tp # an integer + self._generate_gen_const(True, name, check_value=check_value) _loading_gen_macro = _loaded_noop def _loaded_gen_macro(self, tp, name, module, library): - value = self._load_constant(True, tp, name, module) + if tp == '...': + check_value = None + else: + check_value = tp # an integer + value = self._load_constant(True, tp, name, module, + check_value=check_value) setattr(library, name, value) type(library)._cffi_dir.append(name) @@ -565,6 +594,24 @@ cffimod_header = r''' typedef unsigned __int16 uint16_t; typedef unsigned __int32 uint32_t; typedef unsigned __int64 uint64_t; + typedef __int8 int_least8_t; + typedef __int16 int_least16_t; + typedef __int32 int_least32_t; + typedef __int64 int_least64_t; + typedef unsigned __int8 uint_least8_t; + typedef unsigned __int16 uint_least16_t; + typedef unsigned __int32 uint_least32_t; + typedef unsigned __int64 uint_least64_t; + typedef __int8 int_fast8_t; + typedef __int16 int_fast16_t; + typedef __int32 int_fast32_t; + typedef __int64 int_fast64_t; + typedef unsigned __int8 uint_fast8_t; + typedef unsigned __int16 uint_fast16_t; + typedef unsigned __int32 uint_fast32_t; + typedef unsigned __int64 uint_fast64_t; + typedef __int64 intmax_t; + typedef unsigned __int64 uintmax_t; # else # include <stdint.h> # endif diff --git a/lib_pypy/cffi/verifier.py b/lib_pypy/cffi/verifier.py index 9603a7e5ea..ca66be68ca 100644 --- a/lib_pypy/cffi/verifier.py +++ b/lib_pypy/cffi/verifier.py @@ -1,12 +1,23 @@ -import sys, os, binascii, imp, shutil -from . import __version__ +import sys, os, binascii, shutil +from . import __version_verifier_modules__ from . import ffiplatform +if sys.version_info >= (3, 3): + import importlib.machinery + def _extension_suffixes(): + return importlib.machinery.EXTENSION_SUFFIXES[:] +else: + import imp + def _extension_suffixes(): + return [suffix for suffix, _, type in imp.get_suffixes() + if type == imp.C_EXTENSION] + class Verifier(object): def __init__(self, ffi, preamble, tmpdir=None, modulename=None, - ext_package=None, tag='', force_generic_engine=False, **kwds): + ext_package=None, tag='', force_generic_engine=False, + source_extension='.c', flags=None, relative_to=None, **kwds): self.ffi = ffi self.preamble = preamble if not modulename: @@ -14,14 +25,15 @@ class Verifier(object): vengine_class = _locate_engine_class(ffi, force_generic_engine) self._vengine = vengine_class(self) self._vengine.patch_extension_kwds(kwds) - self.kwds = kwds + self.flags = flags + self.kwds = self.make_relative_to(kwds, relative_to) # if modulename: if tag: raise TypeError("can't specify both 'modulename' and 'tag'") else: - key = '\x00'.join([sys.version[:3], __version__, preamble, - flattened_kwds] + + key = '\x00'.join([sys.version[:3], __version_verifier_modules__, + preamble, flattened_kwds] + ffi._cdefsources) if sys.version_info >= (3,): key = key.encode('utf-8') @@ -33,7 +45,7 @@ class Verifier(object): k1, k2) suffix = _get_so_suffixes()[0] self.tmpdir = tmpdir or _caller_dir_pycache() - self.sourcefilename = os.path.join(self.tmpdir, modulename + '.c') + self.sourcefilename = os.path.join(self.tmpdir, modulename + source_extension) self.modulefilename = os.path.join(self.tmpdir, modulename + suffix) self.ext_package = ext_package self._has_source = False @@ -97,6 +109,20 @@ class Verifier(object): def generates_python_module(self): return self._vengine._gen_python_module + def make_relative_to(self, kwds, relative_to): + if relative_to and os.path.dirname(relative_to): + dirname = os.path.dirname(relative_to) + kwds = kwds.copy() + for key in ffiplatform.LIST_OF_FILE_NAMES: + if key in kwds: + lst = kwds[key] + if not isinstance(lst, (list, tuple)): + raise TypeError("keyword '%s' should be a list or tuple" + % (key,)) + lst = [os.path.join(dirname, fn) for fn in lst] + kwds[key] = lst + return kwds + # ---------- def _locate_module(self): @@ -148,7 +174,10 @@ class Verifier(object): def _load_library(self): assert self._has_module - return self._vengine.load_library() + if self.flags is not None: + return self._vengine.load_library(self.flags) + else: + return self._vengine.load_library() # ____________________________________________________________ @@ -181,6 +210,9 @@ _TMPDIR = None def _caller_dir_pycache(): if _TMPDIR: return _TMPDIR + result = os.environ.get('CFFI_TMPDIR') + if result: + return result filename = sys._getframe(2).f_code.co_filename return os.path.abspath(os.path.join(os.path.dirname(filename), '__pycache__')) @@ -222,11 +254,7 @@ def cleanup_tmpdir(tmpdir=None, keep_so=False): pass def _get_so_suffixes(): - suffixes = [] - for suffix, mode, type in imp.get_suffixes(): - if type == imp.C_EXTENSION: - suffixes.append(suffix) - + suffixes = _extension_suffixes() if not suffixes: # bah, no C_EXTENSION available. Occurs on pypy without cpyext if sys.platform == 'win32': diff --git a/pypy/doc/embedding.rst b/pypy/doc/embedding.rst index b4d55e966f..16f0548c95 100644 --- a/pypy/doc/embedding.rst +++ b/pypy/doc/embedding.rst @@ -30,12 +30,10 @@ The API is: Initialize threads. Only need to be called if there are any threads involved -.. function:: long pypy_setup_home(char* home, int verbose); +.. function:: int pypy_setup_home(char* home, int verbose); This function searches the PyPy standard library starting from the given - "PyPy home directory". It is not strictly necessary to execute it before - running Python code, but without it you will not be able to import any - non-builtin module from the standard library. The arguments are: + "PyPy home directory". The arguments are: * ``home``: NULL terminated path to an executable inside the pypy directory (can be a .so name, can be made up) @@ -84,25 +82,36 @@ We write a little C program: const char source[] = "print 'hello from pypy'"; - int main() + int main(void) { - int res; - - rpython_startup_code(); - // pypy_setup_home() is not needed in this trivial example - res = pypy_execute_source((char*)source); - if (res) { - printf("Error calling pypy_execute_source!\n"); - } - return res; + int res; + + rpython_startup_code(); + res = pypy_setup_home("/opt/pypy/bin/libpypy-c.so", 1); + if (res) { + printf("Error setting pypy home!\n"); + return 1; + } + + res = pypy_execute_source((char*)source); + if (res) { + printf("Error calling pypy_execute_source!\n"); + } + return res; } -If we save it as ``x.c`` now, compile it and run it with:: +If we save it as ``x.c`` now, compile it and run it (on linux) with:: fijal@hermann:/opt/pypy$ gcc -o x x.c -lpypy-c -L. fijal@hermann:/opt/pypy$ LD_LIBRARY_PATH=. ./x hello from pypy +on OSX it is necessary to set the rpath of the binary if one wants to link to it:: + + gcc -o x x.c -lpypy-c -L. -Wl,-rpath -Wl,@executable_path + ./x + hello from pypy + Worked! .. note:: If the compilation fails because of missing PyPy.h header file, diff --git a/pypy/doc/install.rst b/pypy/doc/install.rst index 6fcb59b799..9657280aee 100644 --- a/pypy/doc/install.rst +++ b/pypy/doc/install.rst @@ -38,14 +38,13 @@ If you want to make PyPy available system-wide, you can put a symlink to the and not move the binary there, else PyPy would not be able to find its library. -If you want to install 3rd party libraries, the most convenient way is to -install distribute_ and pip_: +If you want to install 3rd party libraries, the most convenient way is +to install pip_ (unless you want to install virtualenv as explained +below; then you can directly use pip inside virtualenvs): .. code-block:: console - $ curl -O http://python-distribute.org/distribute_setup.py - $ curl -O https://raw.githubusercontent.com/pypa/pip/master/contrib/get-pip.py - $ ./pypy-2.1/bin/pypy distribute_setup.py + $ curl -O https://bootstrap.pypa.io/get-pip.py $ ./pypy-2.1/bin/pypy get-pip.py $ ./pypy-2.1/bin/pip install pygments # for example @@ -69,7 +68,6 @@ checkout:: Note that bin/python is now a symlink to bin/pypy. -.. _distribute: http://www.python-distribute.org/ .. _pip: http://pypi.python.org/pypi/pip diff --git a/pypy/goal/targetpypystandalone.py b/pypy/goal/targetpypystandalone.py index 938ca86844..cafbc3657f 100644 --- a/pypy/goal/targetpypystandalone.py +++ b/pypy/goal/targetpypystandalone.py @@ -101,7 +101,7 @@ def create_entry_point(space, w_dict): if space.is_none(w_path): if verbose: debug("Failed to find library based on pypy_find_stdlib") - return 1 + return rffi.cast(rffi.INT, 1) space.startup() space.call_function(w_pathsetter, w_path) # import site @@ -109,13 +109,13 @@ def create_entry_point(space, w_dict): import_ = space.getattr(space.getbuiltinmodule('__builtin__'), space.wrap('__import__')) space.call_function(import_, space.wrap('site')) - return 0 + return rffi.cast(rffi.INT, 0) except OperationError, e: if verbose: debug("OperationError:") debug(" operror-type: " + e.w_type.getname(space)) debug(" operror-value: " + space.str_w(space.str(e.get_w_value(space)))) - return -1 + return rffi.cast(rffi.INT, -1) @entrypoint('main', [rffi.CCHARP], c_name='pypy_execute_source') def pypy_execute_source(ll_source): @@ -234,8 +234,7 @@ class PyPyTarget(object): enable_translationmodules(config) config.translation.suggest(check_str_without_nul=True) - if sys.platform.startswith('linux'): - config.translation.suggest(shared=True) + config.translation.suggest(shared=True) if config.translation.thread: config.objspace.usemodules.thread = True diff --git a/pypy/interpreter/astcompiler/optimize.py b/pypy/interpreter/astcompiler/optimize.py index 98f6d2e065..9833dbdbff 100644 --- a/pypy/interpreter/astcompiler/optimize.py +++ b/pypy/interpreter/astcompiler/optimize.py @@ -83,17 +83,16 @@ class __extend__(ast.UnaryOp): class __extend__(ast.BoolOp): - def _accept_jump_if_any_is(self, gen, condition, target): - self.values[0].accept_jump_if(gen, condition, target) - for i in range(1, len(self.values)): + def _accept_jump_if_any_is(self, gen, condition, target, skip_last=0): + for i in range(len(self.values) - skip_last): self.values[i].accept_jump_if(gen, condition, target) def accept_jump_if(self, gen, condition, target): if condition and self.op == ast.And or \ (not condition and self.op == ast.Or): end = gen.new_block() - self._accept_jump_if_any_is(gen, not condition, end) - gen.emit_jump(ops.JUMP_FORWARD, target) + self._accept_jump_if_any_is(gen, not condition, end, skip_last=1) + self.values[-1].accept_jump_if(gen, condition, target) gen.use_next_block(end) else: self._accept_jump_if_any_is(gen, condition, target) diff --git a/pypy/module/__builtin__/app_io.py b/pypy/module/__builtin__/app_io.py index 83f36968fc..3b25da663c 100644 --- a/pypy/module/__builtin__/app_io.py +++ b/pypy/module/__builtin__/app_io.py @@ -86,9 +86,11 @@ def input(prompt=''): def print_(*args, **kwargs): """The new-style print function from py3k.""" - fp = kwargs.pop("file", sys.stdout) + fp = kwargs.pop("file", None) if fp is None: - return + fp = sys.stdout + if fp is None: + return def write(data): if not isinstance(data, basestring): data = str(data) diff --git a/pypy/module/__builtin__/test/test_builtin.py b/pypy/module/__builtin__/test/test_builtin.py index 6e62643dda..05ffa2d22c 100644 --- a/pypy/module/__builtin__/test/test_builtin.py +++ b/pypy/module/__builtin__/test/test_builtin.py @@ -651,9 +651,12 @@ def fn(): pass out = sys.stdout = StringIO.StringIO() try: pr("Hello,", "person!") + pr("2nd line", file=None) + sys.stdout = None + pr("nowhere") finally: sys.stdout = save - assert out.getvalue() == "Hello, person!\n" + assert out.getvalue() == "Hello, person!\n2nd line\n" out = StringIO.StringIO() pr("Hello,", "person!", file=out) assert out.getvalue() == "Hello, person!\n" @@ -668,7 +671,6 @@ def fn(): pass result = out.getvalue() assert isinstance(result, unicode) assert result == u"Hello, person!\n" - pr("Hello", file=None) # This works. out = StringIO.StringIO() pr(None, file=out) assert out.getvalue() == "None\n" diff --git a/pypy/module/_cffi_backend/__init__.py b/pypy/module/_cffi_backend/__init__.py index a0ceefe8f8..2d02bb1013 100644 --- a/pypy/module/_cffi_backend/__init__.py +++ b/pypy/module/_cffi_backend/__init__.py @@ -34,6 +34,7 @@ class Module(MixedModule): 'newp_handle': 'handle.newp_handle', 'from_handle': 'handle.from_handle', '_get_types': 'func._get_types', + 'from_buffer': 'func.from_buffer', 'string': 'func.string', 'buffer': 'cbuffer.buffer', diff --git a/pypy/module/_cffi_backend/ccallback.py b/pypy/module/_cffi_backend/ccallback.py index 6e4c305c29..ed32731585 100644 --- a/pypy/module/_cffi_backend/ccallback.py +++ b/pypy/module/_cffi_backend/ccallback.py @@ -45,8 +45,9 @@ class W_CDataCallback(W_CData): # cif_descr = self.getfunctype().cif_descr if not cif_descr: - raise OperationError(space.w_NotImplementedError, - space.wrap("callbacks with '...'")) + raise oefmt(space.w_NotImplementedError, + "%s: callback with unsupported argument or " + "return type or with '...'", self.getfunctype().name) res = clibffi.c_ffi_prep_closure(self.get_closure(), cif_descr.cif, invoke_callback, rffi.cast(rffi.VOIDP, self.unique_id)) @@ -98,7 +99,7 @@ class W_CDataCallback(W_CData): def print_error(self, operr, extra_line): space = self.space - operr.write_unraisable(space, "callback ", self.w_callable, + operr.write_unraisable(space, "cffi callback ", self.w_callable, with_traceback=True, extra_line=extra_line) def write_error_return_value(self, ll_res): diff --git a/pypy/module/_cffi_backend/cdataobj.py b/pypy/module/_cffi_backend/cdataobj.py index 6e62ac1e57..8e8f03aa06 100644 --- a/pypy/module/_cffi_backend/cdataobj.py +++ b/pypy/module/_cffi_backend/cdataobj.py @@ -440,6 +440,25 @@ class W_CDataHandle(W_CData): return "handle to %s" % (self.space.str_w(w_repr),) +class W_CDataFromBuffer(W_CData): + _attrs_ = ['buf', 'length', 'w_keepalive'] + _immutable_fields_ = ['buf', 'length', 'w_keepalive'] + + def __init__(self, space, cdata, ctype, buf, w_object): + W_CData.__init__(self, space, cdata, ctype) + self.buf = buf + self.length = buf.getlength() + self.w_keepalive = w_object + + def get_array_length(self): + return self.length + + def _repr_extra(self): + w_repr = self.space.repr(self.w_keepalive) + return "buffer len %d from '%s' object" % ( + self.length, self.space.type(self.w_keepalive).name) + + W_CData.typedef = TypeDef( '_cffi_backend.CData', __module__ = '_cffi_backend', diff --git a/pypy/module/_cffi_backend/ctypearray.py b/pypy/module/_cffi_backend/ctypearray.py index 00f39607a2..10fe6036dc 100644 --- a/pypy/module/_cffi_backend/ctypearray.py +++ b/pypy/module/_cffi_backend/ctypearray.py @@ -107,6 +107,9 @@ class W_CTypeArray(W_CTypePtrOrArray): return self.space.w_None return W_CTypePtrOrArray._fget(self, attrchar) + def typeoffsetof_index(self, index): + return self.ctptr.typeoffsetof_index(index) + class W_CDataIter(W_Root): _immutable_fields_ = ['ctitem', 'cdata', '_stop'] # but not '_next' diff --git a/pypy/module/_cffi_backend/ctypefunc.py b/pypy/module/_cffi_backend/ctypefunc.py index 24591d344e..8fab0cedfc 100644 --- a/pypy/module/_cffi_backend/ctypefunc.py +++ b/pypy/module/_cffi_backend/ctypefunc.py @@ -27,6 +27,8 @@ class W_CTypeFunc(W_CTypePtrBase): _immutable_fields_ = ['fargs[*]', 'ellipsis', 'cif_descr'] kind = "function" + cif_descr = lltype.nullptr(CIF_DESCRIPTION) + def __init__(self, space, fargs, fresult, ellipsis): extra = self._compute_extra_text(fargs, fresult, ellipsis) size = rffi.sizeof(rffi.VOIDP) @@ -41,7 +43,17 @@ class W_CTypeFunc(W_CTypePtrBase): # at all. The cif is computed on every call from the actual # types passed in. For all other functions, the cif_descr # is computed here. - CifDescrBuilder(fargs, fresult).rawallocate(self) + builder = CifDescrBuilder(fargs, fresult) + try: + builder.rawallocate(self) + except OperationError, e: + if not e.match(space, space.w_NotImplementedError): + raise + # else, eat the NotImplementedError. We will get the + # exception if we see an actual call + if self.cif_descr: # should not be True, but you never know + lltype.free(self.cif_descr, flavor='raw') + self.cif_descr = lltype.nullptr(CIF_DESCRIPTION) def new_ctypefunc_completing_argtypes(self, args_w): space = self.space @@ -57,10 +69,12 @@ class W_CTypeFunc(W_CTypePtrBase): "argument %d passed in the variadic part needs to " "be a cdata object (got %T)", i + 1, w_obj) fvarargs[i] = ct + # xxx call instantiate() directly. It's a bit of a hack. ctypefunc = instantiate(W_CTypeFunc) ctypefunc.space = space ctypefunc.fargs = fvarargs ctypefunc.ctitem = self.ctitem + #ctypefunc.cif_descr = NULL --- already provided as the default CifDescrBuilder(fvarargs, self.ctitem).rawallocate(ctypefunc) return ctypefunc @@ -178,8 +192,6 @@ def _get_abi(space, name): # ____________________________________________________________ -W_CTypeFunc.cif_descr = lltype.nullptr(CIF_DESCRIPTION) # default value - BIG_ENDIAN = sys.byteorder == 'big' USE_C_LIBFFI_MSVC = getattr(clibffi, 'USE_C_LIBFFI_MSVC', False) @@ -295,18 +307,18 @@ class CifDescrBuilder(object): nflat = 0 for i, cf in enumerate(ctype.fields_list): if cf.is_bitfield(): - raise OperationError(space.w_NotImplementedError, - space.wrap("cannot pass as argument or return value " - "a struct with bit fields")) + raise oefmt(space.w_NotImplementedError, + "ctype '%s' not supported as argument or return value" + " (it is a struct with bit fields)", ctype.name) flat = 1 ct = cf.ctype while isinstance(ct, ctypearray.W_CTypeArray): flat *= ct.length ct = ct.ctitem if flat <= 0: - raise OperationError(space.w_NotImplementedError, - space.wrap("cannot pass as argument or return value " - "a struct with a zero-length array")) + raise oefmt(space.w_NotImplementedError, + "ctype '%s' not supported as argument or return value" + " (it is a struct with a zero-length array)", ctype.name) nflat += flat if USE_C_LIBFFI_MSVC and is_result_type: diff --git a/pypy/module/_cffi_backend/ctypeobj.py b/pypy/module/_cffi_backend/ctypeobj.py index 5750ed954a..3fd7706b8e 100644 --- a/pypy/module/_cffi_backend/ctypeobj.py +++ b/pypy/module/_cffi_backend/ctypeobj.py @@ -142,12 +142,14 @@ class W_CType(W_Root): raise oefmt(space.w_ValueError, "ctype '%s' is of unknown alignment", self.name) - def typeoffsetof(self, fieldname): + def typeoffsetof_field(self, fieldname, following): space = self.space - if fieldname is None: - msg = "expected a struct or union ctype" - else: - msg = "expected a struct or union ctype, or a pointer to one" + msg = "with a field name argument, expected a struct or union ctype" + raise OperationError(space.w_TypeError, space.wrap(msg)) + + def typeoffsetof_index(self, index): + space = self.space + msg = "with an integer argument, expected an array or pointer ctype" raise OperationError(space.w_TypeError, space.wrap(msg)) def rawaddressof(self, cdata, offset): diff --git a/pypy/module/_cffi_backend/ctypeptr.py b/pypy/module/_cffi_backend/ctypeptr.py index 83ab7a3d84..60fe50cd5c 100644 --- a/pypy/module/_cffi_backend/ctypeptr.py +++ b/pypy/module/_cffi_backend/ctypeptr.py @@ -308,24 +308,36 @@ class W_CTypePointer(W_CTypePtrBase): def getcfield(self, attr): return self.ctitem.getcfield(attr) - def typeoffsetof(self, fieldname): - if fieldname is None: - return W_CTypePtrBase.typeoffsetof(self, fieldname) - else: - return self.ctitem.typeoffsetof(fieldname) + def typeoffsetof_field(self, fieldname, following): + if following == 0: + return self.ctitem.typeoffsetof_field(fieldname, -1) + return W_CTypePtrBase.typeoffsetof_field(self, fieldname, following) + + def typeoffsetof_index(self, index): + space = self.space + ctitem = self.ctitem + if ctitem.size < 0: + raise OperationError(space.w_TypeError, + space.wrap("pointer to opaque")) + try: + offset = ovfcheck(index * ctitem.size) + except OverflowError: + raise OperationError(space.w_OverflowError, + space.wrap("array offset would overflow a ssize_t")) + return ctitem, offset def rawaddressof(self, cdata, offset): from pypy.module._cffi_backend.ctypestruct import W_CTypeStructOrUnion space = self.space ctype2 = cdata.ctype if (isinstance(ctype2, W_CTypeStructOrUnion) or - (isinstance(ctype2, W_CTypePtrOrArray) and - isinstance(ctype2.ctitem, W_CTypeStructOrUnion))): + isinstance(ctype2, W_CTypePtrOrArray)): ptrdata = rffi.ptradd(cdata._cdata, offset) return cdataobj.W_CData(space, ptrdata, self) else: raise OperationError(space.w_TypeError, - space.wrap("expected a 'cdata struct-or-union' object")) + space.wrap("expected a cdata struct/union/array/pointer" + " object")) def _fget(self, attrchar): if attrchar == 'i': # item diff --git a/pypy/module/_cffi_backend/ctypestruct.py b/pypy/module/_cffi_backend/ctypestruct.py index c2d169fd4b..bc189be93b 100644 --- a/pypy/module/_cffi_backend/ctypestruct.py +++ b/pypy/module/_cffi_backend/ctypestruct.py @@ -65,9 +65,7 @@ class W_CTypeStructOrUnion(W_CType): keepalive_until_here(ob) return ob - def typeoffsetof(self, fieldname): - if fieldname is None: - return (self, 0) + def typeoffsetof_field(self, fieldname, following): self.check_complete() space = self.space try: diff --git a/pypy/module/_cffi_backend/func.py b/pypy/module/_cffi_backend/func.py index 99019578b8..8649750265 100644 --- a/pypy/module/_cffi_backend/func.py +++ b/pypy/module/_cffi_backend/func.py @@ -48,13 +48,28 @@ def alignof(space, w_ctype): align = w_ctype.alignof() return space.wrap(align) -@unwrap_spec(w_ctype=ctypeobj.W_CType, fieldname="str_or_None") -def typeoffsetof(space, w_ctype, fieldname): - ctype, offset = w_ctype.typeoffsetof(fieldname) +@unwrap_spec(w_ctype=ctypeobj.W_CType, following=int) +def typeoffsetof(space, w_ctype, w_field_or_index, following=0): + try: + fieldname = space.str_w(w_field_or_index) + except OperationError, e: + if not e.match(space, space.w_TypeError): + raise + try: + index = space.int_w(w_field_or_index) + except OperationError, e: + if not e.match(space, space.w_TypeError): + raise + raise OperationError(space.w_TypeError, + space.wrap("field name or array index expected")) + ctype, offset = w_ctype.typeoffsetof_index(index) + else: + ctype, offset = w_ctype.typeoffsetof_field(fieldname, following) + # return space.newtuple([space.wrap(ctype), space.wrap(offset)]) @unwrap_spec(w_ctype=ctypeobj.W_CType, w_cdata=cdataobj.W_CData, offset=int) -def rawaddressof(space, w_ctype, w_cdata, offset=0): +def rawaddressof(space, w_ctype, w_cdata, offset): return w_ctype.rawaddressof(w_cdata, offset) # ____________________________________________________________ @@ -76,3 +91,32 @@ def string(space, w_cdata, maxlen=-1): def _get_types(space): return space.newtuple([space.gettypefor(cdataobj.W_CData), space.gettypefor(ctypeobj.W_CType)]) + +# ____________________________________________________________ + +@unwrap_spec(w_ctype=ctypeobj.W_CType) +def from_buffer(space, w_ctype, w_x): + from pypy.module._cffi_backend import ctypearray, ctypeprim + # + if (not isinstance(w_ctype, ctypearray.W_CTypeArray) or + not isinstance(w_ctype.ctptr.ctitem, ctypeprim.W_CTypePrimitiveChar)): + raise oefmt(space.w_TypeError, + "needs 'char[]', got '%s'", w_ctype.name) + # + # xxx do we really need to implement the same mess as in CPython 2.7 + # w.r.t. buffers and memoryviews?? + try: + buf = space.readbuf_w(w_x) + except OperationError, e: + if not e.match(space, space.w_TypeError): + raise + buf = space.buffer_w(w_x, space.BUF_SIMPLE) + try: + _cdata = buf.get_raw_address() + except ValueError: + raise oefmt(space.w_TypeError, + "from_buffer() got a '%T' object, which supports the " + "buffer interface but cannot be rendered as a plain " + "raw address on PyPy", w_x) + # + return cdataobj.W_CDataFromBuffer(space, _cdata, w_ctype, buf, w_x) diff --git a/pypy/module/_cffi_backend/newtype.py b/pypy/module/_cffi_backend/newtype.py index 6ca8077256..a4c75b039b 100644 --- a/pypy/module/_cffi_backend/newtype.py +++ b/pypy/module/_cffi_backend/newtype.py @@ -62,10 +62,54 @@ eptypesize("uint64_t", 8, ctypeprim.W_CTypePrimitiveUnsigned) eptype("intptr_t", rffi.INTPTR_T, ctypeprim.W_CTypePrimitiveSigned) eptype("uintptr_t", rffi.UINTPTR_T, ctypeprim.W_CTypePrimitiveUnsigned) -eptype("ptrdiff_t", rffi.INTPTR_T, ctypeprim.W_CTypePrimitiveSigned) # <-xxx eptype("size_t", rffi.SIZE_T, ctypeprim.W_CTypePrimitiveUnsigned) eptype("ssize_t", rffi.SSIZE_T, ctypeprim.W_CTypePrimitiveSigned) +_WCTSigned = ctypeprim.W_CTypePrimitiveSigned +_WCTUnsign = ctypeprim.W_CTypePrimitiveUnsigned + +eptype("ptrdiff_t", getattr(rffi, 'PTRDIFF_T', rffi.INTPTR_T), _WCTSigned) +eptype("intmax_t", getattr(rffi, 'INTMAX_T', rffi.LONGLONG), _WCTSigned) +eptype("uintmax_t", getattr(rffi, 'UINTMAX_T', rffi.LONGLONG), _WCTUnsign) + +if hasattr(rffi, 'INT_LEAST8_T'): + eptype("int_least8_t", rffi.INT_LEAST8_T, _WCTSigned) + eptype("int_least16_t", rffi.INT_LEAST16_T, _WCTSigned) + eptype("int_least32_t", rffi.INT_LEAST32_T, _WCTSigned) + eptype("int_least64_t", rffi.INT_LEAST64_T, _WCTSigned) + eptype("uint_least8_t", rffi.UINT_LEAST8_T, _WCTUnsign) + eptype("uint_least16_t",rffi.UINT_LEAST16_T, _WCTUnsign) + eptype("uint_least32_t",rffi.UINT_LEAST32_T, _WCTUnsign) + eptype("uint_least64_t",rffi.UINT_LEAST64_T, _WCTUnsign) +else: + eptypesize("int_least8_t", 1, _WCTSigned) + eptypesize("uint_least8_t", 1, _WCTUnsign) + eptypesize("int_least16_t", 2, _WCTSigned) + eptypesize("uint_least16_t", 2, _WCTUnsign) + eptypesize("int_least32_t", 4, _WCTSigned) + eptypesize("uint_least32_t", 4, _WCTUnsign) + eptypesize("int_least64_t", 8, _WCTSigned) + eptypesize("uint_least64_t", 8, _WCTUnsign) + +if hasattr(rffi, 'INT_FAST8_T'): + eptype("int_fast8_t", rffi.INT_FAST8_T, _WCTSigned) + eptype("int_fast16_t", rffi.INT_FAST16_T, _WCTSigned) + eptype("int_fast32_t", rffi.INT_FAST32_T, _WCTSigned) + eptype("int_fast64_t", rffi.INT_FAST64_T, _WCTSigned) + eptype("uint_fast8_t", rffi.UINT_FAST8_T, _WCTUnsign) + eptype("uint_fast16_t",rffi.UINT_FAST16_T, _WCTUnsign) + eptype("uint_fast32_t",rffi.UINT_FAST32_T, _WCTUnsign) + eptype("uint_fast64_t",rffi.UINT_FAST64_T, _WCTUnsign) +else: + eptypesize("int_fast8_t", 1, _WCTSigned) + eptypesize("uint_fast8_t", 1, _WCTUnsign) + eptypesize("int_fast16_t", 2, _WCTSigned) + eptypesize("uint_fast16_t", 2, _WCTUnsign) + eptypesize("int_fast32_t", 4, _WCTSigned) + eptypesize("uint_fast32_t", 4, _WCTUnsign) + eptypesize("int_fast64_t", 8, _WCTSigned) + eptypesize("uint_fast64_t", 8, _WCTUnsign) + @unwrap_spec(name=str) def new_primitive_type(space, name): try: diff --git a/pypy/module/_cffi_backend/test/_backend_test_c.py b/pypy/module/_cffi_backend/test/_backend_test_c.py index 715385ba31..77994393d5 100644 --- a/pypy/module/_cffi_backend/test/_backend_test_c.py +++ b/pypy/module/_cffi_backend/test/_backend_test_c.py @@ -1030,11 +1030,12 @@ def test_cannot_pass_struct_with_array_of_length_0(): BInt = new_primitive_type("int") BArray0 = new_array_type(new_pointer_type(BInt), 0) BStruct = new_struct_type("struct foo") + BStructP = new_pointer_type(BStruct) complete_struct_or_union(BStruct, [('a', BArray0)]) - py.test.raises(NotImplementedError, new_function_type, - (BStruct,), BInt, False) - py.test.raises(NotImplementedError, new_function_type, - (BInt,), BStruct, False) + BFunc = new_function_type((BStruct,), BInt, False) + py.test.raises(NotImplementedError, cast(BFunc, 123), cast(BStructP, 123)) + BFunc2 = new_function_type((BInt,), BStruct, False) + py.test.raises(NotImplementedError, cast(BFunc2, 123), 123) def test_call_function_9(): BInt = new_primitive_type("int") @@ -1174,7 +1175,7 @@ def test_callback_exception(): assert sys.stderr.getvalue() == '' assert f(10000) == -42 assert matches(sys.stderr.getvalue(), """\ -From callback <function$Zcb1 at 0x$>: +From cffi callback <function$Zcb1 at 0x$>: Traceback (most recent call last): File "$", line $, in Zcb1 $ @@ -1186,7 +1187,7 @@ ValueError: 42 bigvalue = 20000 assert f(bigvalue) == -42 assert matches(sys.stderr.getvalue(), """\ -From callback <function$Zcb1 at 0x$>: +From cffi callback <function$Zcb1 at 0x$>: Trying to convert the result back to C: OverflowError: integer 60000 does not fit 'short' """) @@ -1805,7 +1806,8 @@ def test_invalid_function_result_types(): new_function_type((), new_pointer_type(BFunc)) BUnion = new_union_type("union foo_u") complete_struct_or_union(BUnion, []) - py.test.raises(NotImplementedError, new_function_type, (), BUnion) + BFunc = new_function_type((), BUnion) + py.test.raises(NotImplementedError, cast(BFunc, 123)) py.test.raises(TypeError, new_function_type, (), BArray) def test_struct_return_in_func(): @@ -2525,13 +2527,32 @@ def test_typeoffsetof(): ('a2', BChar, -1), ('a3', BChar, -1)]) py.test.raises(TypeError, typeoffsetof, BStructPtr, None) - assert typeoffsetof(BStruct, None) == (BStruct, 0) + py.test.raises(TypeError, typeoffsetof, BStruct, None) assert typeoffsetof(BStructPtr, 'a1') == (BChar, 0) assert typeoffsetof(BStruct, 'a1') == (BChar, 0) assert typeoffsetof(BStructPtr, 'a2') == (BChar, 1) assert typeoffsetof(BStruct, 'a3') == (BChar, 2) + assert typeoffsetof(BStructPtr, 'a2', 0) == (BChar, 1) + assert typeoffsetof(BStruct, u+'a3') == (BChar, 2) + py.test.raises(TypeError, typeoffsetof, BStructPtr, 'a2', 1) py.test.raises(KeyError, typeoffsetof, BStructPtr, 'a4') py.test.raises(KeyError, typeoffsetof, BStruct, 'a5') + py.test.raises(TypeError, typeoffsetof, BStruct, 42) + py.test.raises(TypeError, typeoffsetof, BChar, 'a1') + +def test_typeoffsetof_array(): + BInt = new_primitive_type("int") + BIntP = new_pointer_type(BInt) + BArray = new_array_type(BIntP, None) + py.test.raises(TypeError, typeoffsetof, BArray, None) + py.test.raises(TypeError, typeoffsetof, BArray, 'a1') + assert typeoffsetof(BArray, 51) == (BInt, 51 * size_of_int()) + assert typeoffsetof(BIntP, 51) == (BInt, 51 * size_of_int()) + assert typeoffsetof(BArray, -51) == (BInt, -51 * size_of_int()) + MAX = sys.maxsize // size_of_int() + assert typeoffsetof(BArray, MAX) == (BInt, MAX * size_of_int()) + assert typeoffsetof(BIntP, MAX) == (BInt, MAX * size_of_int()) + py.test.raises(OverflowError, typeoffsetof, BArray, MAX + 1) def test_typeoffsetof_no_bitfield(): BInt = new_primitive_type("int") @@ -2551,17 +2572,26 @@ def test_rawaddressof(): assert repr(p) == "<cdata 'struct foo *' owning 3 bytes>" s = p[0] assert repr(s) == "<cdata 'struct foo' owning 3 bytes>" - a = rawaddressof(BStructPtr, s) + a = rawaddressof(BStructPtr, s, 0) assert repr(a).startswith("<cdata 'struct foo *' 0x") - py.test.raises(TypeError, rawaddressof, BStruct, s) - b = rawaddressof(BCharP, s) + py.test.raises(TypeError, rawaddressof, BStruct, s, 0) + b = rawaddressof(BCharP, s, 0) assert b == cast(BCharP, p) - c = rawaddressof(BStructPtr, a) + c = rawaddressof(BStructPtr, a, 0) assert c == a - py.test.raises(TypeError, rawaddressof, BStructPtr, cast(BChar, '?')) + py.test.raises(TypeError, rawaddressof, BStructPtr, cast(BChar, '?'), 0) # d = rawaddressof(BCharP, s, 1) assert d == cast(BCharP, p) + 1 + # + e = cast(BCharP, 109238) + f = rawaddressof(BCharP, e, 42) + assert f == e + 42 + # + BCharA = new_array_type(BCharP, None) + e = newp(BCharA, 50) + f = rawaddressof(BCharP, e, 42) + assert f == e + 42 def test_newp_signed_unsigned_char(): BCharArray = new_array_type( @@ -2718,7 +2748,16 @@ def test_GetLastError(): def test_nonstandard_integer_types(): for typename in ['int8_t', 'uint8_t', 'int16_t', 'uint16_t', 'int32_t', 'uint32_t', 'int64_t', 'uint64_t', 'intptr_t', - 'uintptr_t', 'ptrdiff_t', 'size_t', 'ssize_t']: + 'uintptr_t', 'ptrdiff_t', 'size_t', 'ssize_t', + 'int_least8_t', 'uint_least8_t', + 'int_least16_t', 'uint_least16_t', + 'int_least32_t', 'uint_least32_t', + 'int_least64_t', 'uint_least64_t', + 'int_fast8_t', 'uint_fast8_t', + 'int_fast16_t', 'uint_fast16_t', + 'int_fast32_t', 'uint_fast32_t', + 'int_fast64_t', 'uint_fast64_t', + 'intmax_t', 'uintmax_t']: new_primitive_type(typename) # works def test_cannot_convert_unicode_to_charp(): @@ -3186,6 +3225,20 @@ def test_packed_with_bitfields(): ('a2', BChar, 5)], None, -1, -1, SF_PACKED) +def test_from_buffer(): + import array + a = array.array('H', [10000, 20000, 30000]) + BChar = new_primitive_type("char") + BCharP = new_pointer_type(BChar) + BCharA = new_array_type(BCharP, None) + c = from_buffer(BCharA, a) + assert typeof(c) is BCharA + assert len(c) == 6 + assert repr(c) == "<cdata 'char[]' buffer len 6 from 'array.array' object>" + p = new_pointer_type(new_primitive_type("unsigned short")) + cast(p, c)[1] += 500 + assert list(a) == [10000, 20500, 30000] + def test_version(): # this test is here mostly for PyPy assert __version__ == "0.8.6" diff --git a/pypy/module/_cffi_backend/test/test_c.py b/pypy/module/_cffi_backend/test/test_c.py index 3d95b01978..13022f287e 100644 --- a/pypy/module/_cffi_backend/test/test_c.py +++ b/pypy/module/_cffi_backend/test/test_c.py @@ -30,7 +30,7 @@ from rpython.translator.tool.cbuild import ExternalCompilationInfo class AppTestC(object): """Populated below, hack hack hack.""" - spaceconfig = dict(usemodules=('_cffi_backend', 'cStringIO')) + spaceconfig = dict(usemodules=('_cffi_backend', 'cStringIO', 'array')) def setup_class(cls): testfuncs_w = [] diff --git a/pypy/module/_io/interp_bufferedio.py b/pypy/module/_io/interp_bufferedio.py index 11164d101f..c4be1d9cea 100644 --- a/pypy/module/_io/interp_bufferedio.py +++ b/pypy/module/_io/interp_bufferedio.py @@ -565,7 +565,7 @@ class BufferedMixin: # Flush the write buffer if necessary if self.writable: - self._writer_flush_unlocked(space) + self._flush_and_rewind_unlocked(space) self._reader_reset_buf() # Read whole blocks, and don't buffer them diff --git a/pypy/module/_io/test/test_io.py b/pypy/module/_io/test/test_io.py index e4b4f6157c..6107d01bef 100644 --- a/pypy/module/_io/test/test_io.py +++ b/pypy/module/_io/test/test_io.py @@ -362,3 +362,32 @@ class AppTestOpen: f.read(1) f.seek(-1, 1) f.write(b'') + + def test_issue1902_2(self): + import _io + with _io.open(self.tmpfile, 'w+b', 4096) as f: + f.write(b'\xff' * 13569) + f.flush() + f.seek(0, 0) + + f.read(1) + f.seek(-1, 1) + f.write(b'\xff') + f.seek(1, 0) + f.read(4123) + f.seek(-4123, 1) + + def test_issue1902_3(self): + import _io + buffer_size = 4096 + with _io.open(self.tmpfile, 'w+b', buffer_size) as f: + f.write(b'\xff' * buffer_size * 3) + f.flush() + f.seek(0, 0) + + f.read(1) + f.seek(-1, 1) + f.write(b'\xff') + f.seek(1, 0) + f.read(buffer_size * 2) + assert f.tell() == 1 + buffer_size * 2 diff --git a/pypy/module/_rawffi/buffer.py b/pypy/module/_rawffi/buffer.py index 4c63ede9b0..50c881977f 100644 --- a/pypy/module/_rawffi/buffer.py +++ b/pypy/module/_rawffi/buffer.py @@ -1,4 +1,5 @@ from rpython.rlib.buffer import Buffer +from rpython.rtyper.lltypesystem import rffi # XXX not the most efficient implementation @@ -20,3 +21,7 @@ class RawFFIBuffer(Buffer): def setitem(self, index, char): ll_buffer = self.datainstance.ll_buffer ll_buffer[index] = char + + def get_raw_address(self): + ll_buffer = self.datainstance.ll_buffer + return rffi.cast(rffi.CCHARP, ll_buffer) diff --git a/pypy/module/_rawffi/test/test__rawffi.py b/pypy/module/_rawffi/test/test__rawffi.py index ebc53f2be7..1aa6590a0f 100644 --- a/pypy/module/_rawffi/test/test__rawffi.py +++ b/pypy/module/_rawffi/test/test__rawffi.py @@ -1144,6 +1144,15 @@ class AppTestFfi: b[3] = b'x' assert b[3] == b'x' + def test_pypy_raw_address(self): + import _rawffi + S = _rawffi.Structure((40, 1)) + s = S(autofree=True) + addr = buffer(s)._pypy_raw_address() + assert type(addr) is int + assert buffer(s)._pypy_raw_address() == addr + assert buffer(s, 10)._pypy_raw_address() == addr + 10 + def test_union(self): import _rawffi longsize = _rawffi.sizeof('l') diff --git a/pypy/module/cpyext/slotdefs.py b/pypy/module/cpyext/slotdefs.py index 11af7c1f2f..27fee27d23 100644 --- a/pypy/module/cpyext/slotdefs.py +++ b/pypy/module/cpyext/slotdefs.py @@ -244,6 +244,9 @@ class CPyBuffer(Buffer): def getitem(self, index): return self.ptr[index] + def get_raw_address(self): + return rffi.cast(rffi.CCHARP, self.ptr) + def wrap_getreadbuffer(space, w_self, w_args, func): func_target = rffi.cast(readbufferproc, func) with lltype.scoped_alloc(rffi.VOIDPP.TO, 1) as ptr: diff --git a/pypy/module/gc/__init__.py b/pypy/module/gc/__init__.py index cd0bd181d9..8fa139698e 100644 --- a/pypy/module/gc/__init__.py +++ b/pypy/module/gc/__init__.py @@ -30,6 +30,7 @@ class Module(MixedModule): 'get_referrers': 'referents.get_referrers', '_dump_rpy_heap': 'referents._dump_rpy_heap', 'get_typeids_z': 'referents.get_typeids_z', + 'get_typeids_list': 'referents.get_typeids_list', 'GcRef': 'referents.W_GcRef', }) MixedModule.__init__(self, space, w_name) diff --git a/pypy/module/gc/app_referents.py b/pypy/module/gc/app_referents.py index b97a5baa05..570280c9a4 100644 --- a/pypy/module/gc/app_referents.py +++ b/pypy/module/gc/app_referents.py @@ -16,7 +16,8 @@ def dump_rpy_heap(file): [0][0][0][-1] inserted after all GC roots, before all non-roots. If the argument is a filename and the 'zlib' module is available, - we also write a 'typeids.txt' in the same directory, if none exists. + we also write 'typeids.txt' and 'typeids.lst' in the same directory, + if they don't already exist. """ if isinstance(file, str): f = open(file, 'wb') @@ -30,7 +31,13 @@ def dump_rpy_heap(file): filename2 = os.path.join(os.path.dirname(file), 'typeids.txt') if not os.path.exists(filename2): data = zlib.decompress(gc.get_typeids_z()) - f = open(filename2, 'wb') + f = open(filename2, 'w') + f.write(data) + f.close() + filename2 = os.path.join(os.path.dirname(file), 'typeids.lst') + if not os.path.exists(filename2): + data = ''.join(['%d\n' % n for n in gc.get_typeids_list()]) + f = open(filename2, 'w') f.write(data) f.close() else: diff --git a/pypy/module/gc/referents.py b/pypy/module/gc/referents.py index 19f4462aa7..bf6e24a763 100644 --- a/pypy/module/gc/referents.py +++ b/pypy/module/gc/referents.py @@ -228,3 +228,8 @@ def get_typeids_z(space): a = rgc.get_typeids_z() s = ''.join([a[i] for i in range(len(a))]) return space.wrap(s) + +def get_typeids_list(space): + l = rgc.get_typeids_list() + list_w = [space.wrap(l[i]) for i in range(len(l))] + return space.newlist(list_w) diff --git a/pypy/module/micronumpy/descriptor.py b/pypy/module/micronumpy/descriptor.py index fe9889f184..e4a9c7c33b 100644 --- a/pypy/module/micronumpy/descriptor.py +++ b/pypy/module/micronumpy/descriptor.py @@ -392,7 +392,7 @@ class W_Dtype(W_Root): alignment = space.int_w(space.getitem(w_data, space.wrap(6))) if (w_names == space.w_None) != (w_fields == space.w_None): - raise oefmt(space.w_ValueError, "inconsistent fields and names") + raise oefmt(space.w_ValueError, "inconsistent fields and names in Numpy dtype unpickling") self.byteorder = endian self.shape = [] diff --git a/pypy/module/micronumpy/test/test_complex.py b/pypy/module/micronumpy/test/test_complex.py index 888527545d..b009fcf926 100644 --- a/pypy/module/micronumpy/test/test_complex.py +++ b/pypy/module/micronumpy/test/test_complex.py @@ -478,6 +478,15 @@ class AppTestUfuncs(BaseNumpyAppTest): for i in range(4): assert c[i] == max(a[i], b[i]) + + def test_abs_overflow(self): + from numpy import array, absolute, isinf + a = array(complex(1.5e308,1.5e308)) + # Prints a RuntimeWarning, but does not raise + b = absolute(a) + assert isinf(b) + + def test_basic(self): import sys from numpy import (dtype, add, array, dtype, diff --git a/pypy/module/micronumpy/test/test_dtypes.py b/pypy/module/micronumpy/test/test_dtypes.py index 6d537e2f37..30a216c0a1 100644 --- a/pypy/module/micronumpy/test/test_dtypes.py +++ b/pypy/module/micronumpy/test/test_dtypes.py @@ -665,6 +665,7 @@ class AppTestTypes(BaseAppTestDtypes): assert numpy.int64(9223372036854775807) == 9223372036854775807 assert numpy.int64(9223372036854775807) == 9223372036854775807 + assert numpy.int64(-9223372036854775807) == -9223372036854775807 raises(OverflowError, numpy.int64, 9223372036854775808) raises(OverflowError, numpy.int64, 9223372036854775808L) @@ -1233,7 +1234,8 @@ class AppTestRecordDtypes(BaseNumpyAppTest): d = np.dtype(('<f8', 2)) exc = raises(ValueError, "d.__setstate__((3, '|', None, ('f0', 'f1'), None, 16, 1, 0))") - assert exc.value[0] == 'inconsistent fields and names' + inconsistent = 'inconsistent fields and names in Numpy dtype unpickling' + assert exc.value[0] == inconsistent assert d.fields is None assert d.shape == (2,) assert d.subdtype is not None @@ -1241,7 +1243,7 @@ class AppTestRecordDtypes(BaseNumpyAppTest): d = np.dtype(('<f8', 2)) exc = raises(ValueError, "d.__setstate__((3, '|', None, None, {'f0': (np.dtype('float64'), 0), 'f1': (np.dtype('float64'), 8)}, 16, 1, 0))") - assert exc.value[0] == 'inconsistent fields and names' + assert exc.value[0] == inconsistent assert d.fields is None assert d.shape == (2,) assert d.subdtype is not None @@ -1282,7 +1284,11 @@ class AppTestRecordDtypes(BaseNumpyAppTest): from cPickle import loads, dumps d = dtype([("x", "int32"), ("y", "int32"), ("z", "int32"), ("value", float)]) - assert d.__reduce__() == (dtype, ('V20', 0, 1), (3, '|', None, ('x', 'y', 'z', 'value'), {'y': (dtype('int32'), 4), 'x': (dtype('int32'), 0), 'z': (dtype('int32'), 8), 'value': (dtype('float64'), 12)}, 20, 1, 0)) + assert d.__reduce__() == (dtype, ('V20', 0, 1), (3, '|', None, + ('x', 'y', 'z', 'value'), + {'y': (dtype('int32'), 4), 'x': (dtype('int32'), 0), + 'z': (dtype('int32'), 8), 'value': (dtype('float64'), 12), + }, 20, 1, 0)) new_d = loads(dumps(d)) diff --git a/pypy/module/micronumpy/tool/numready/main.py b/pypy/module/micronumpy/tool/numready/main.py index 844cae97e2..321ebd94fe 100644 --- a/pypy/module/micronumpy/tool/numready/main.py +++ b/pypy/module/micronumpy/tool/numready/main.py @@ -71,6 +71,10 @@ def find_numpy_items(python, modname="numpy", attr=None): lines = subprocess.check_output(args).splitlines() items = SearchableSet() for line in lines: + # since calling a function in "search.py" may have printed side effects, + # make sure the line begins with '[UT] : ' + if not (line[:1] in KINDS.values() and line[1:4] == ' : '): + continue kind, name = line.split(" : ", 1) subitems = [] if kind == KINDS["TYPE"] and name in SPECIAL_NAMES and attr is None: @@ -97,7 +101,15 @@ SPECIAL_NAMES = ["ndarray", "dtype", "generic", "flatiter", "ufunc", "nditer"] def main(argv): - cpy_items = find_numpy_items("/usr/bin/python") + if 'help' in argv[1]: + print '\nusage: python', os.path.dirname(__file__), '<path-to-pypy> [<outfile.html>] [<path-to-cpython-with-numpy>]' + print ' path-to-cpython-with-numpy defaults to "/usr/bin/python"\n' + return + if len(argv) < 4: + cpython = '/usr/bin/python' + else: + cpython = argv[3] + cpy_items = find_numpy_items(cpython) pypy_items = find_numpy_items(argv[1]) ver = get_version_str(argv[1]) all_items = [] diff --git a/pypy/module/micronumpy/tool/numready/search.py b/pypy/module/micronumpy/tool/numready/search.py index 967aad0a41..39a2dbffff 100644 --- a/pypy/module/micronumpy/tool/numready/search.py +++ b/pypy/module/micronumpy/tool/numready/search.py @@ -23,6 +23,15 @@ def main(argv): if attr is None and name.startswith("_"): continue subobj = getattr(obj, name) + if subobj is None: + continue + if isinstance(subobj, types.FunctionType): + try: + subobj() + except NotImplementedError: + continue + except: + pass if isinstance(subobj, types.TypeType): kind = KINDS["TYPE"] else: diff --git a/pypy/module/micronumpy/types.py b/pypy/module/micronumpy/types.py index 3365fc8912..73ee74602c 100644 --- a/pypy/module/micronumpy/types.py +++ b/pypy/module/micronumpy/types.py @@ -1195,7 +1195,11 @@ class ComplexFloating(object): @complex_to_real_unary_op def abs(self, v): - return rcomplex.c_abs(v[0], v[1]) + try: + return rcomplex.c_abs(v[0], v[1]) + except OverflowError: + # warning ... + return rfloat.INFINITY @raw_unary_op def isnan(self, v): diff --git a/pypy/module/micronumpy/ufuncs.py b/pypy/module/micronumpy/ufuncs.py index 360e3aed85..70681d64bd 100644 --- a/pypy/module/micronumpy/ufuncs.py +++ b/pypy/module/micronumpy/ufuncs.py @@ -619,6 +619,9 @@ def find_dtype_for_scalar(space, w_obj, current_guess=None): space.int_w(w_obj) except OperationError, e: if e.match(space, space.w_OverflowError): + if space.is_true(space.le(w_obj, space.wrap(0))): + return find_binop_result_dtype(space, int64_dtype, + current_guess) return find_binop_result_dtype(space, uint64_dtype, current_guess) raise diff --git a/pypy/module/select/test/test_select.py b/pypy/module/select/test/test_select.py index 609f16e7b9..9094f51829 100644 --- a/pypy/module/select/test/test_select.py +++ b/pypy/module/select/test/test_select.py @@ -286,7 +286,7 @@ class AppTestSelectWithPipes(_AppTestSelect): t = thread.start_new_thread(pollster.poll, ()) try: - time.sleep(0.1) + time.sleep(0.3) for i in range(5): print '', # to release GIL untranslated # trigger ufds array reallocation for fd in rfds: @@ -297,7 +297,7 @@ class AppTestSelectWithPipes(_AppTestSelect): finally: # and make the call to poll() from the thread return os.write(w, b'spam') - time.sleep(0.1) + time.sleep(0.3) for i in range(5): print '', # to release GIL untranslated finally: os.close(r) diff --git a/pypy/module/termios/__init__.py b/pypy/module/termios/__init__.py index a519899777..34f2f672bc 100644 --- a/pypy/module/termios/__init__.py +++ b/pypy/module/termios/__init__.py @@ -1,5 +1,7 @@ from pypy.interpreter.mixedmodule import MixedModule +from rpython.rlib import rtermios + class Module(MixedModule): "This module provides an interface to the Posix calls for tty I/O control.\n\ For a complete description of these calls, see the Posix or Unix manual\n\ @@ -23,10 +25,6 @@ class Module(MixedModule): 'error' : 'space.fromcache(interp_termios.Cache).w_error', } -# XXX this is extremaly not-portable, but how to prevent this? - -import termios -for i in dir(termios): - val = getattr(termios, i) - if i.isupper() and type(val) is int: - Module.interpleveldefs[i] = "space.wrap(%s)" % val + for name in rtermios.all_constants: + value = getattr(rtermios, name) + interpleveldefs[name] = "space.wrap(%s)" % value diff --git a/pypy/module/termios/interp_termios.py b/pypy/module/termios/interp_termios.py index aed17f0114..179c3d6fac 100644 --- a/pypy/module/termios/interp_termios.py +++ b/pypy/module/termios/interp_termios.py @@ -6,7 +6,6 @@ little use of termios module on RPython level by itself from pypy.interpreter.gateway import unwrap_spec from pypy.interpreter.error import wrap_oserror, OperationError from rpython.rlib import rtermios -import termios class Cache: def __init__(self, space): @@ -52,9 +51,9 @@ def tcgetattr(space, w_fd): l_w = [space.wrap(i) for i in [iflag, oflag, cflag, lflag, ispeed, ospeed]] # last one need to be chosen carefully cc_w = [space.wrap(i) for i in cc] - if lflag & termios.ICANON: - cc_w[termios.VMIN] = space.wrap(ord(cc[termios.VMIN][0])) - cc_w[termios.VTIME] = space.wrap(ord(cc[termios.VTIME][0])) + if lflag & rtermios.ICANON: + cc_w[rtermios.VMIN] = space.wrap(ord(cc[rtermios.VMIN][0])) + cc_w[rtermios.VTIME] = space.wrap(ord(cc[rtermios.VTIME][0])) w_cc = space.newlist(cc_w) l_w.append(w_cc) return space.newlist(l_w) @@ -63,14 +62,14 @@ def tcgetattr(space, w_fd): def tcsendbreak(space, w_fd, duration): fd = space.c_filedescriptor_w(w_fd) try: - termios.tcsendbreak(fd, duration) + rtermios.tcsendbreak(fd, duration) except OSError, e: raise convert_error(space, e) def tcdrain(space, w_fd): fd = space.c_filedescriptor_w(w_fd) try: - termios.tcdrain(fd) + rtermios.tcdrain(fd) except OSError, e: raise convert_error(space, e) @@ -78,7 +77,7 @@ def tcdrain(space, w_fd): def tcflush(space, w_fd, queue): fd = space.c_filedescriptor_w(w_fd) try: - termios.tcflush(fd, queue) + rtermios.tcflush(fd, queue) except OSError, e: raise convert_error(space, e) @@ -86,6 +85,6 @@ def tcflush(space, w_fd, queue): def tcflow(space, w_fd, action): fd = space.c_filedescriptor_w(w_fd) try: - termios.tcflow(fd, action) + rtermios.tcflow(fd, action) except OSError, e: raise convert_error(space, e) diff --git a/pypy/module/termios/test/test_termios.py b/pypy/module/termios/test/test_termios.py index 42a6aa9589..2a51ea673e 100644 --- a/pypy/module/termios/test/test_termios.py +++ b/pypy/module/termios/test/test_termios.py @@ -136,7 +136,7 @@ class AppTestTermios(object): val = getattr(termios, name) if name.isupper() and type(val) is int: d[name] = val - assert d == self.orig_module_dict + assert sorted(d.items()) == sorted(self.orig_module_dict.items()) def test_error(self): import termios, errno, os diff --git a/pypy/module/test_lib_pypy/cffi_tests/backend_tests.py b/pypy/module/test_lib_pypy/cffi_tests/backend_tests.py index ee8238e78b..8345ab8fa5 100644 --- a/pypy/module/test_lib_pypy/cffi_tests/backend_tests.py +++ b/pypy/module/test_lib_pypy/cffi_tests/backend_tests.py @@ -2,7 +2,7 @@ import py import platform import sys, ctypes -from cffi import FFI, CDefError +from cffi import FFI, CDefError, FFIError from pypy.module.test_lib_pypy.cffi_tests.support import * SIZE_OF_INT = ctypes.sizeof(ctypes.c_int) @@ -917,6 +917,16 @@ class BackendTests: assert int(invalid_value) == 2 assert ffi.string(invalid_value) == "2" + def test_enum_char_hex_oct(self): + ffi = FFI(backend=self.Backend()) + ffi.cdef(r"enum foo{A='!', B='\'', C=0x10, D=010, E=- 0x10, F=-010};") + assert ffi.string(ffi.cast("enum foo", ord('!'))) == "A" + assert ffi.string(ffi.cast("enum foo", ord("'"))) == "B" + assert ffi.string(ffi.cast("enum foo", 16)) == "C" + assert ffi.string(ffi.cast("enum foo", 8)) == "D" + assert ffi.string(ffi.cast("enum foo", -16)) == "E" + assert ffi.string(ffi.cast("enum foo", -8)) == "F" + def test_array_of_struct(self): ffi = FFI(backend=self.Backend()) ffi.cdef("struct foo { int a, b; };") @@ -950,6 +960,25 @@ class BackendTests: assert ffi.offsetof("struct foo", "b") == 4 assert ffi.offsetof("struct foo", "c") == 8 + def test_offsetof_nested(self): + ffi = FFI(backend=self.Backend()) + ffi.cdef("struct foo { int a, b, c; };" + "struct bar { struct foo d, e; };") + assert ffi.offsetof("struct bar", "e") == 12 + py.test.raises(KeyError, ffi.offsetof, "struct bar", "e.a") + assert ffi.offsetof("struct bar", "e", "a") == 12 + assert ffi.offsetof("struct bar", "e", "b") == 16 + assert ffi.offsetof("struct bar", "e", "c") == 20 + + def test_offsetof_array(self): + ffi = FFI(backend=self.Backend()) + assert ffi.offsetof("int[]", 51) == 51 * ffi.sizeof("int") + assert ffi.offsetof("int *", 51) == 51 * ffi.sizeof("int") + ffi.cdef("struct bar { int a, b; int c[99]; };") + assert ffi.offsetof("struct bar", "c") == 2 * ffi.sizeof("int") + assert ffi.offsetof("struct bar", "c", 0) == 2 * ffi.sizeof("int") + assert ffi.offsetof("struct bar", "c", 51) == 53 * ffi.sizeof("int") + def test_alignof(self): ffi = FFI(backend=self.Backend()) ffi.cdef("struct foo { char a; short b; char c; };") @@ -1482,8 +1511,10 @@ class BackendTests: p = ffi.new("struct foo_s *") a = ffi.addressof(p[0]) assert repr(a).startswith("<cdata 'struct foo_s *' 0x") + assert a == p py.test.raises(TypeError, ffi.addressof, p) py.test.raises((AttributeError, TypeError), ffi.addressof, 5) + py.test.raises(TypeError, ffi.addressof, ffi.cast("int", 5)) def test_addressof_field(self): ffi = FFI(backend=self.Backend()) @@ -1496,6 +1527,17 @@ class BackendTests: assert a == ffi.addressof(p, 'y') assert a != ffi.addressof(p, 'x') + def test_addressof_field_nested(self): + ffi = FFI(backend=self.Backend()) + ffi.cdef("struct foo_s { int x, y; };" + "struct bar_s { struct foo_s a, b; };") + p = ffi.new("struct bar_s *") + py.test.raises(KeyError, ffi.addressof, p[0], 'b.y') + a = ffi.addressof(p[0], 'b', 'y') + assert int(ffi.cast("uintptr_t", a)) == ( + int(ffi.cast("uintptr_t", p)) + + ffi.sizeof("struct foo_s") + ffi.sizeof("int")) + def test_addressof_anonymous_struct(self): ffi = FFI() ffi.cdef("typedef struct { int x; } foo_t;") @@ -1503,6 +1545,49 @@ class BackendTests: a = ffi.addressof(p[0]) assert a == p + def test_addressof_array(self): + ffi = FFI() + p = ffi.new("int[52]") + p0 = ffi.addressof(p) + assert p0 == p + assert ffi.typeof(p0) is ffi.typeof("int(*)[52]") + py.test.raises(TypeError, ffi.addressof, p0) + # + p1 = ffi.addressof(p, 25) + assert ffi.typeof(p1) is ffi.typeof("int *") + assert (p1 - p) == 25 + assert ffi.addressof(p, 0) == p + + def test_addressof_pointer(self): + ffi = FFI() + array = ffi.new("int[50]") + p = ffi.cast("int *", array) + py.test.raises(TypeError, ffi.addressof, p) + assert ffi.addressof(p, 0) == p + assert ffi.addressof(p, 25) == p + 25 + assert ffi.typeof(ffi.addressof(p, 25)) == ffi.typeof(p) + # + ffi.cdef("struct foo { int a, b; };") + array = ffi.new("struct foo[50]") + p = ffi.cast("int *", array) + py.test.raises(TypeError, ffi.addressof, p) + assert ffi.addressof(p, 0) == p + assert ffi.addressof(p, 25) == p + 25 + assert ffi.typeof(ffi.addressof(p, 25)) == ffi.typeof(p) + + def test_addressof_array_in_struct(self): + ffi = FFI() + ffi.cdef("struct foo { int a, b; int c[50]; };") + p = ffi.new("struct foo *") + p1 = ffi.addressof(p, "c", 25) + assert ffi.typeof(p1) is ffi.typeof("int *") + assert p1 == ffi.cast("int *", p) + 27 + assert ffi.addressof(p, "c") == ffi.cast("int *", p) + 2 + assert ffi.addressof(p, "c", 0) == ffi.cast("int *", p) + 2 + p2 = ffi.addressof(p, 1) + assert ffi.typeof(p2) is ffi.typeof("struct foo *") + assert p2 == p + 1 + def test_multiple_independent_structs(self): ffi1 = FFI(); ffi1.cdef("struct foo { int x; };") ffi2 = FFI(); ffi2.cdef("struct foo { int y, z; };") @@ -1565,6 +1650,12 @@ class BackendTests: p = ffi2.new("foo_p", [142]) assert p.x == 142 + def test_ignore_multiple_declarations_of_constant(self): + ffi = FFI(backend=self.Backend()) + ffi.cdef("#define FOO 42") + ffi.cdef("#define FOO 42") + py.test.raises(FFIError, ffi.cdef, "#define FOO 43") + def test_struct_packed(self): ffi = FFI(backend=self.Backend()) ffi.cdef("struct nonpacked { char a; int b; };") diff --git a/pypy/module/test_lib_pypy/cffi_tests/test_cdata.py b/pypy/module/test_lib_pypy/cffi_tests/test_cdata.py index 51e039ac72..8dc9b98598 100644 --- a/pypy/module/test_lib_pypy/cffi_tests/test_cdata.py +++ b/pypy/module/test_lib_pypy/cffi_tests/test_cdata.py @@ -20,6 +20,8 @@ class FakeBackend(object): return FakeType("void") def new_pointer_type(self, x): return FakeType('ptr-to-%r' % (x,)) + def new_array_type(self, x, y): + return FakeType('array-from-%r-len-%r' % (x, y)) def cast(self, x, y): return 'casted!' def _get_types(self): diff --git a/pypy/module/test_lib_pypy/cffi_tests/test_ffi_backend.py b/pypy/module/test_lib_pypy/cffi_tests/test_ffi_backend.py index 3873de19ab..0f36e966c1 100644 --- a/pypy/module/test_lib_pypy/cffi_tests/test_ffi_backend.py +++ b/pypy/module/test_lib_pypy/cffi_tests/test_ffi_backend.py @@ -20,8 +20,8 @@ class TestFFI(backend_tests.BackendTests, ffi.cdef("struct foo_s { int a,b,c,d,e; int x:1; };") e = py.test.raises(NotImplementedError, ffi.callback, "struct foo_s foo(void)", lambda: 42) - assert str(e.value) == ("<struct foo_s(*)(void)>: " - "cannot pass as argument or return value a struct with bit fields") + assert str(e.value) == ("struct foo_s(*)(): " + "callback with unsupported argument or return type or with '...'") def test_inspecttype(self): ffi = FFI(backend=self.Backend()) @@ -123,7 +123,7 @@ class TestBitfield: self.check("int a:2; short b:15; char c:2; char y;", 5, 4, 8) self.check("int a:2; char b:1; char c:1; char y;", 1, 4, 4) - @pytest.mark.skipif("platform.machine().startswith('arm')") + @pytest.mark.skipif("platform.machine().startswith(('arm', 'aarch64'))") def test_bitfield_anonymous_no_align(self): L = FFI().alignof("long long") self.check("char y; int :1;", 0, 1, 2) @@ -136,7 +136,8 @@ class TestBitfield: self.check("char x; long long z:57; char y;", L + 8, L, L + 8 + L) self.check("char x; long long :57; char y;", L + 8, 1, L + 9) - @pytest.mark.skipif("not platform.machine().startswith('arm')") + @pytest.mark.skipif( + "not platform.machine().startswith(('arm', 'aarch64'))") def test_bitfield_anonymous_align_arm(self): L = FFI().alignof("long long") self.check("char y; int :1;", 0, 4, 4) @@ -149,7 +150,7 @@ class TestBitfield: self.check("char x; long long z:57; char y;", L + 8, L, L + 8 + L) self.check("char x; long long :57; char y;", L + 8, L, L + 8 + L) - @pytest.mark.skipif("platform.machine().startswith('arm')") + @pytest.mark.skipif("platform.machine().startswith(('arm', 'aarch64'))") def test_bitfield_zero(self): L = FFI().alignof("long long") self.check("char y; int :0;", 0, 1, 4) @@ -160,7 +161,8 @@ class TestBitfield: self.check("char x; int :0; short b:1; char y;", 5, 2, 6) self.check("int a:1; int :0; int b:1; char y;", 5, 4, 8) - @pytest.mark.skipif("not platform.machine().startswith('arm')") + @pytest.mark.skipif( + "not platform.machine().startswith(('arm', 'aarch64'))") def test_bitfield_zero_arm(self): L = FFI().alignof("long long") self.check("char y; int :0;", 0, 4, 4) @@ -212,3 +214,12 @@ class TestBitfield: code, message = ffi.getwinerror(-1) assert code == 2 assert message == "The system cannot find the file specified" + + def test_from_buffer(self): + import array + ffi = FFI() + a = array.array('H', [10000, 20000, 30000]) + c = ffi.from_buffer(a) + assert ffi.typeof(c) is ffi.typeof("char[]") + ffi.cast("unsigned short *", c)[1] += 500 + assert list(a) == [10000, 20500, 30000] diff --git a/pypy/module/test_lib_pypy/cffi_tests/test_parsing.py b/pypy/module/test_lib_pypy/cffi_tests/test_parsing.py index b514177d0c..4512ad7e88 100644 --- a/pypy/module/test_lib_pypy/cffi_tests/test_parsing.py +++ b/pypy/module/test_lib_pypy/cffi_tests/test_parsing.py @@ -164,8 +164,12 @@ def test_define_not_supported_for_now(): ffi = FFI(backend=FakeBackend()) e = py.test.raises(CDefError, ffi.cdef, '#define FOO "blah"') assert str(e.value) == ( - 'only supports the syntax "#define FOO ..." (literally)' - ' or "#define FOO 0x1FF" for now') + 'only supports one of the following syntax:\n' + ' #define FOO ... (literally dot-dot-dot)\n' + ' #define FOO NUMBER (with NUMBER an integer' + ' constant, decimal/hex/octal)\n' + 'got:\n' + ' #define FOO "blah"') def test_unnamed_struct(): ffi = FFI(backend=FakeBackend()) @@ -248,7 +252,8 @@ def test_win_common_types(): ct = win_common_types(maxsize) clear_all(ct) for key in sorted(ct): - resolve_common_type(key) + if ct[key] != 'set-unicode-needed': + resolve_common_type(key) # assert did not crash # now try to use e.g. WPARAM (-> UINT_PTR -> unsigned 32/64-bit) for maxsize in [2**32-1, 2**64-1]: @@ -289,3 +294,14 @@ def test__is_constant_globalvar(): decl = ast.children()[0][1] node = decl.type assert p._is_constant_globalvar(node) == expected_output + +def test_enum(): + ffi = FFI() + ffi.cdef(""" + enum Enum { POS = +1, TWO = 2, NIL = 0, NEG = -1}; + """) + C = ffi.dlopen(None) + assert C.POS == 1 + assert C.TWO == 2 + assert C.NIL == 0 + assert C.NEG == -1 diff --git a/pypy/module/test_lib_pypy/cffi_tests/test_verify.py b/pypy/module/test_lib_pypy/cffi_tests/test_verify.py index 1fdde0c73c..7ff36341a9 100644 --- a/pypy/module/test_lib_pypy/cffi_tests/test_verify.py +++ b/pypy/module/test_lib_pypy/cffi_tests/test_verify.py @@ -1,7 +1,7 @@ # Generated by pypy/tool/import_cffi.py import py, re import sys, os, math, weakref -from cffi import FFI, VerificationError, VerificationMissing, model +from cffi import FFI, VerificationError, VerificationMissing, model, FFIError from pypy.module.test_lib_pypy.cffi_tests.support import * @@ -15,12 +15,13 @@ if sys.platform == 'win32': else: if (sys.platform == 'darwin' and [int(x) for x in os.uname()[2].split('.')] >= [11, 0, 0]): + # assume a standard clang or gcc + extra_compile_args = ['-Werror', '-Wall', '-Wextra', '-Wconversion'] # special things for clang - extra_compile_args = [ - '-Werror', '-Qunused-arguments', '-Wno-error=shorten-64-to-32'] + extra_compile_args.append('-Qunused-arguments') else: # assume a standard gcc - extra_compile_args = ['-Werror'] + extra_compile_args = ['-Werror', '-Wall', '-Wextra', '-Wconversion'] class FFI(FFI): def verify(self, *args, **kwds): @@ -90,11 +91,48 @@ def test_simple_case(): lib = ffi.verify('#include <math.h>', libraries=lib_m) assert lib.sin(1.23) == math.sin(1.23) +def _Wconversion(cdef, source, **kargs): + if sys.platform == 'win32': + py.test.skip("needs GCC or Clang") + ffi = FFI() + ffi.cdef(cdef) + py.test.raises(VerificationError, ffi.verify, source, **kargs) + extra_compile_args_orig = extra_compile_args[:] + extra_compile_args.remove('-Wconversion') + try: + lib = ffi.verify(source, **kargs) + finally: + extra_compile_args[:] = extra_compile_args_orig + return lib + +def test_Wconversion_unsigned(): + _Wconversion("unsigned foo(void);", + "int foo(void) { return -1;}") + +def test_Wconversion_integer(): + _Wconversion("short foo(void);", + "long long foo(void) { return 1<<sizeof(short);}") + +def test_Wconversion_floating(): + lib = _Wconversion("float sin(double);", + "#include <math.h>", libraries=lib_m) + res = lib.sin(1.23) + assert res != math.sin(1.23) # not exact, because of double->float + assert abs(res - math.sin(1.23)) < 1E-5 + +def test_Wconversion_float2int(): + _Wconversion("int sinf(float);", + "#include <math.h>", libraries=lib_m) + +def test_Wconversion_double2int(): + _Wconversion("int sin(double);", + "#include <math.h>", libraries=lib_m) + def test_rounding_1(): ffi = FFI() - ffi.cdef("float sin(double x);") + ffi.cdef("double sinf(float x);") lib = ffi.verify('#include <math.h>', libraries=lib_m) - res = lib.sin(1.23) + res = lib.sinf(1.23) assert res != math.sin(1.23) # not exact, because of double->float assert abs(res - math.sin(1.23)) < 1E-5 @@ -113,14 +151,21 @@ def test_strlen_exact(): assert lib.strlen(b"hi there!") == 9 def test_strlen_approximate(): - ffi = FFI() - ffi.cdef("int strlen(char *s);") - lib = ffi.verify("#include <string.h>") + lib = _Wconversion("int strlen(char *s);", + "#include <string.h>") assert lib.strlen(b"hi there!") == 9 +def test_return_approximate(): + for typename in ['short', 'int', 'long', 'long long']: + ffi = FFI() + ffi.cdef("%s foo(signed char x);" % typename) + lib = ffi.verify("signed char foo(signed char x) { return x;}") + assert lib.foo(-128) == -128 + assert lib.foo(+127) == +127 + def test_strlen_array_of_char(): ffi = FFI() - ffi.cdef("int strlen(char[]);") + ffi.cdef("size_t strlen(char[]);") lib = ffi.verify("#include <string.h>") assert lib.strlen(b"hello") == 5 @@ -209,8 +254,8 @@ def test_all_integer_and_float_types(): ffi = FFI() ffi.cdef('\n'.join(["%s foo_%s(%s);" % (tp, tp.replace(' ', '_'), tp) for tp in typenames])) - lib = ffi.verify('\n'.join(["%s foo_%s(%s x) { return x+1; }" % - (tp, tp.replace(' ', '_'), tp) + lib = ffi.verify('\n'.join(["%s foo_%s(%s x) { return (%s)(x+1); }" % + (tp, tp.replace(' ', '_'), tp, tp) for tp in typenames])) for typename in typenames: foo = getattr(lib, 'foo_%s' % typename.replace(' ', '_')) @@ -316,7 +361,7 @@ def test_fn_unsigned_integer_types(): def test_char_type(): ffi = FFI() ffi.cdef("char foo(char);") - lib = ffi.verify("char foo(char x) { return x+1; }") + lib = ffi.verify("char foo(char x) { return ++x; }") assert lib.foo(b"A") == b"B" py.test.raises(TypeError, lib.foo, b"bar") py.test.raises(TypeError, lib.foo, "bar") @@ -386,7 +431,7 @@ def test_nondecl_struct(): ffi = FFI() ffi.cdef("typedef struct foo_s foo_t; int bar(foo_t *);") lib = ffi.verify("typedef struct foo_s foo_t;\n" - "int bar(foo_t *f) { return 42; }\n") + "int bar(foo_t *f) { (void)f; return 42; }\n") assert lib.bar(ffi.NULL) == 42 def test_ffi_full_struct(): @@ -897,7 +942,7 @@ def test_unknown_type(): static int foo(token_t *tk) { if (!tk) return -42; - *tk += 1.601; + *tk += 1.601f; return (int)*tk; } #define TOKEN_SIZE sizeof(token_t) @@ -992,7 +1037,7 @@ def test_autofilled_struct_as_argument(): long a; }; int foo(struct foo_s s) { - return s.a - (int)s.b; + return (int)s.a - (int)s.b; } """) s = ffi.new("struct foo_s *", [100, 1]) @@ -1009,7 +1054,7 @@ def test_autofilled_struct_as_argument_dynamic(): long a; }; int foo1(struct foo_s s) { - return s.a - (int)s.b; + return (int)s.a - (int)s.b; } int (*foo)(struct foo_s s) = &foo1; """) @@ -1068,7 +1113,7 @@ def test_func_as_argument(): def test_array_as_argument(): ffi = FFI() ffi.cdef(""" - int strlen(char string[]); + size_t strlen(char string[]); """) ffi.verify("#include <string.h>") @@ -1080,7 +1125,7 @@ def test_enum_as_argument(): """) lib = ffi.verify(""" enum foo_e { AA, CC, BB }; - int foo_func(enum foo_e e) { return e; } + int foo_func(enum foo_e e) { return (int)e; } """) assert lib.foo_func(lib.BB) == 2 py.test.raises(TypeError, lib.foo_func, "BB") @@ -1093,7 +1138,7 @@ def test_enum_as_function_result(): """) lib = ffi.verify(""" enum foo_e { AA, CC, BB }; - enum foo_e foo_func(int x) { return x; } + enum foo_e foo_func(int x) { return (enum foo_e)x; } """) assert lib.foo_func(lib.BB) == lib.BB == 2 @@ -1128,6 +1173,19 @@ def test_typedef_incomplete_enum(): assert lib.AA == 0 assert lib.BB == 2 +def test_typedef_enum_as_argument(): + ffi = FFI() + ffi.cdef(""" + typedef enum { AA, BB, ... } foo_t; + int foo_func(foo_t); + """) + lib = ffi.verify(""" + typedef enum { AA, CC, BB } foo_t; + int foo_func(foo_t e) { return (int)e; } + """) + assert lib.foo_func(lib.BB) == lib.BB == 2 + py.test.raises(TypeError, lib.foo_func, "BB") + def test_typedef_enum_as_function_result(): ffi = FFI() ffi.cdef(""" @@ -1136,7 +1194,7 @@ def test_typedef_enum_as_function_result(): """) lib = ffi.verify(""" typedef enum { AA, CC, BB } foo_t; - foo_t foo_func(int x) { return x; } + foo_t foo_func(int x) { return (foo_t)x; } """) assert lib.foo_func(lib.BB) == lib.BB == 2 @@ -1163,6 +1221,8 @@ def test_opaque_integer_as_function_result(): import platform if platform.machine().startswith('sparc'): py.test.skip('Breaks horribly on sparc (SIGILL + corrupted stack)') + elif platform.machine() == 'mips64' and sys.maxsize > 2**32: + py.test.skip('Segfaults on mips64el') # XXX bad abuse of "struct { ...; }". It only works a bit by chance # anyway. XXX think about something better :-( ffi = FFI() @@ -1292,7 +1352,7 @@ def test_ffi_struct_packed(): """) def test_tmpdir(): - import tempfile, os, shutil + import tempfile, os from pypy.module.test_lib_pypy.cffi_tests.udir import udir tmpdir = tempfile.mkdtemp(dir=str(udir)) ffi = FFI() @@ -1301,6 +1361,20 @@ def test_tmpdir(): assert os.listdir(tmpdir) assert lib.foo(100) == 142 +def test_relative_to(): + import tempfile, os + from pypy.module.test_lib_pypy.cffi_tests.udir import udir + tmpdir = tempfile.mkdtemp(dir=str(udir)) + ffi = FFI() + ffi.cdef("int foo(int);") + f = open(os.path.join(tmpdir, 'foo.h'), 'w') + print >> f, "int foo(int a) { return a + 42; }" + f.close() + lib = ffi.verify('#include "foo.h"', + include_dirs=['.'], + relative_to=os.path.join(tmpdir, 'x')) + assert lib.foo(100) == 142 + def test_bug1(): ffi = FFI() ffi.cdef(""" @@ -1677,7 +1751,7 @@ def test_callback_indirection(): static int c_callback(int how_many, ...) { va_list ap; /* collect the "..." arguments into the values[] array */ - int i, *values = alloca(how_many * sizeof(int)); + int i, *values = alloca((size_t)how_many * sizeof(int)); va_start(ap, how_many); for (i=0; i<how_many; i++) values[i] = va_arg(ap, int); @@ -1718,7 +1792,7 @@ def test_charstar_argument(): ffi.cdef("char sum3chars(char *);") lib = ffi.verify(""" char sum3chars(char *f) { - return f[0] + f[1] + f[2]; + return (char)(f[0] + f[1] + f[2]); } """) assert lib.sum3chars((b'\x10', b'\x20', b'\x30')) == b'\x60' @@ -1753,7 +1827,7 @@ def test_passing_string_or_NULL(): def test_typeof_function(): ffi = FFI() ffi.cdef("int foo(int, char);") - lib = ffi.verify("int foo(int x, char y) { return 42; }") + lib = ffi.verify("int foo(int x, char y) { (void)x; (void)y; return 42; }") ctype = ffi.typeof(lib.foo) assert len(ctype.args) == 2 assert ctype.result == ffi.typeof("int") @@ -1818,6 +1892,7 @@ def _test_various_calls(force_libffi): long tf_bl(signed char x, long c); unsigned long tf_bL(signed char x, unsigned long c); long long tf_bq(signed char x, long long c); + unsigned long long tf_bQ(signed char x, unsigned long long c); float tf_bf(signed char x, float c); double tf_bd(signed char x, double c); long double tf_bD(signed char x, long double c); @@ -1835,20 +1910,35 @@ def _test_various_calls(force_libffi): double dvalue; long double Dvalue; - #define S(letter) xvalue = x; letter##value = c; return rvalue; - - signed char tf_bb(signed char x, signed char c) { S(i) } - unsigned char tf_bB(signed char x, unsigned char c) { S(i) } - short tf_bh(signed char x, short c) { S(i) } - unsigned short tf_bH(signed char x, unsigned short c) { S(i) } - int tf_bi(signed char x, int c) { S(i) } - unsigned int tf_bI(signed char x, unsigned int c) { S(i) } - long tf_bl(signed char x, long c) { S(i) } - unsigned long tf_bL(signed char x, unsigned long c) { S(i) } - long long tf_bq(signed char x, long long c) { S(i) } - float tf_bf(signed char x, float c) { S(f) } - double tf_bd(signed char x, double c) { S(d) } - long double tf_bD(signed char x, long double c) { S(D) } + typedef signed char b_t; + typedef unsigned char B_t; + typedef short h_t; + typedef unsigned short H_t; + typedef int i_t; + typedef unsigned int I_t; + typedef long l_t; + typedef unsigned long L_t; + typedef long long q_t; + typedef unsigned long long Q_t; + typedef float f_t; + typedef double d_t; + typedef long double D_t; + #define S(letter) xvalue = (int)x; letter##value = (letter##_t)c; + #define R(letter) return (letter##_t)rvalue; + + signed char tf_bb(signed char x, signed char c) { S(i) R(b) } + unsigned char tf_bB(signed char x, unsigned char c) { S(i) R(B) } + short tf_bh(signed char x, short c) { S(i) R(h) } + unsigned short tf_bH(signed char x, unsigned short c) { S(i) R(H) } + int tf_bi(signed char x, int c) { S(i) R(i) } + unsigned int tf_bI(signed char x, unsigned int c) { S(i) R(I) } + long tf_bl(signed char x, long c) { S(i) R(l) } + unsigned long tf_bL(signed char x, unsigned long c) { S(i) R(L) } + long long tf_bq(signed char x, long long c) { S(i) R(q) } + unsigned long long tf_bQ(signed char x, unsigned long long c) { S(i) R(Q) } + float tf_bf(signed char x, float c) { S(f) R(f) } + double tf_bd(signed char x, double c) { S(d) R(d) } + long double tf_bD(signed char x, long double c) { S(D) R(D) } """) lib.rvalue = 0x7182838485868788 for kind, cname in [('b', 'signed char'), @@ -1860,6 +1950,7 @@ def _test_various_calls(force_libffi): ('l', 'long'), ('L', 'unsigned long'), ('q', 'long long'), + ('Q', 'unsigned long long'), ('f', 'float'), ('d', 'double'), ('D', 'long double')]: @@ -1963,3 +2054,101 @@ def test_getlasterror_working_even_with_pypys_jit(): n = (1 << 29) + i lib.SetLastError(n) assert ffi.getwinerror()[0] == n + +def test_verify_dlopen_flags(): + # Careful with RTLD_GLOBAL. If by chance the FFI is not deleted + # promptly, like on PyPy, then other tests may see the same + # exported symbols as well. So we must not export a simple name + # like 'foo'! + ffi1 = FFI() + ffi1.cdef("int foo_verify_dlopen_flags;") + + lib1 = ffi1.verify("int foo_verify_dlopen_flags;", + flags=ffi1.RTLD_GLOBAL | ffi1.RTLD_LAZY) + lib2 = get_second_lib() + + lib1.foo_verify_dlopen_flags = 42 + assert lib2.foo_verify_dlopen_flags == 42 + lib2.foo_verify_dlopen_flags += 1 + assert lib1.foo_verify_dlopen_flags == 43 + +def get_second_lib(): + # Hack, using modulename makes the test fail + ffi2 = FFI() + ffi2.cdef("int foo_verify_dlopen_flags;") + lib2 = ffi2.verify("int foo_verify_dlopen_flags;", + flags=ffi2.RTLD_GLOBAL | ffi2.RTLD_LAZY) + return lib2 + +def test_consider_not_implemented_function_type(): + ffi = FFI() + ffi.cdef("typedef union { int a; float b; } Data;" + "typedef struct { int a:2; } MyStr;" + "typedef void (*foofunc_t)(Data);" + "typedef Data (*bazfunc_t)(void);" + "typedef MyStr (*barfunc_t)(void);") + fooptr = ffi.cast("foofunc_t", 123) + bazptr = ffi.cast("bazfunc_t", 123) + barptr = ffi.cast("barfunc_t", 123) + # assert did not crash so far + e = py.test.raises(NotImplementedError, fooptr, ffi.new("Data *")) + assert str(e.value) == ( + "ctype 'Data' (size 4) not supported as argument") + e = py.test.raises(NotImplementedError, bazptr) + assert str(e.value) == ( + "ctype 'Data' (size 4) not supported as return value") + e = py.test.raises(NotImplementedError, barptr) + assert str(e.value) == ( + "ctype 'MyStr' not supported as argument or return value " + "(it is a struct with bit fields)") + +def test_verify_extra_arguments(): + ffi = FFI() + ffi.cdef("#define ABA ...") + lib = ffi.verify("", define_macros=[('ABA', '42')]) + assert lib.ABA == 42 + +def test_implicit_unicode_on_windows(): + if sys.platform != 'win32': + py.test.skip("win32-only test") + ffi = FFI() + e = py.test.raises(FFIError, ffi.cdef, "int foo(LPTSTR);") + assert str(e.value) == ("The Windows type 'LPTSTR' is only available after" + " you call ffi.set_unicode()") + for with_unicode in [True, False]: + ffi = FFI() + ffi.set_unicode(with_unicode) + ffi.cdef(""" + DWORD GetModuleFileName(HMODULE hModule, LPTSTR lpFilename, + DWORD nSize); + """) + lib = ffi.verify(""" + #include <windows.h> + """, libraries=['Kernel32']) + outbuf = ffi.new("TCHAR[]", 200) + n = lib.GetModuleFileName(ffi.NULL, outbuf, 500) + assert 0 < n < 500 + for i in range(n): + print repr(outbuf[i]) + assert ord(outbuf[i]) != 0 + assert ord(outbuf[n]) == 0 + assert ord(outbuf[0]) < 128 # should be a letter, or '\' + +def test_use_local_dir(): + ffi = FFI() + lib = ffi.verify("", modulename="test_use_local_dir") + this_dir = os.path.dirname(__file__) + pycache_files = os.listdir(os.path.join(this_dir, '__pycache__')) + assert any('test_use_local_dir' in s for s in pycache_files) + +def test_define_known_value(): + ffi = FFI() + ffi.cdef("#define FOO 0x123") + lib = ffi.verify("#define FOO 0x123") + assert lib.FOO == 0x123 + +def test_define_wrong_value(): + ffi = FFI() + ffi.cdef("#define FOO 123") + e = py.test.raises(VerificationError, ffi.verify, "#define FOO 124") + assert str(e.value).endswith("FOO has the real value 124, not 123") diff --git a/pypy/module/test_lib_pypy/cffi_tests/test_zdistutils.py b/pypy/module/test_lib_pypy/cffi_tests/test_zdistutils.py index 16572b2fcf..eb49abf508 100644 --- a/pypy/module/test_lib_pypy/cffi_tests/test_zdistutils.py +++ b/pypy/module/test_lib_pypy/cffi_tests/test_zdistutils.py @@ -18,7 +18,7 @@ class DistUtilsTest(object): def teardown_class(self): if udir.isdir(): - udir.remove() + udir.remove(ignore_errors=True) def test_locate_engine_class(self): cls = _locate_engine_class(FFI(), self.generic) diff --git a/pypy/module/test_lib_pypy/cffi_tests/test_zintegration.py b/pypy/module/test_lib_pypy/cffi_tests/test_zintegration.py index f2cf54bbf5..57654ced8a 100644 --- a/pypy/module/test_lib_pypy/cffi_tests/test_zintegration.py +++ b/pypy/module/test_lib_pypy/cffi_tests/test_zintegration.py @@ -76,7 +76,7 @@ def run_setup_and_program(dirname, python_snippet): class TestZIntegration(object): def teardown_class(self): if udir.isdir(): - udir.remove() + udir.remove(ignore_errors=True) def test_infrastructure(self): run_setup_and_program('infrastructure', ''' diff --git a/pypy/module/thread/gil.py b/pypy/module/thread/gil.py index 00835a162c..496d6457dd 100644 --- a/pypy/module/thread/gil.py +++ b/pypy/module/thread/gil.py @@ -7,7 +7,7 @@ Global Interpreter Lock. # all but one will be blocked. The other threads get a chance to run # from time to time, using the periodic action GILReleaseAction. -from rpython.rlib import rthread, rgil +from rpython.rlib import rthread, rgil, rwin32 from pypy.module.thread.error import wrap_thread_error from pypy.interpreter.executioncontext import PeriodicAsyncAction from pypy.module.thread.threadlocals import OSThreadLocals @@ -76,9 +76,14 @@ before_external_call._dont_reach_me_in_del_ = True def after_external_call(): e = get_errno() + e2 = 0 + if rwin32.WIN32: + e2 = rwin32.GetLastError() rgil.gil_acquire() rthread.gc_thread_run() after_thread_switch() + if rwin32.WIN32: + rwin32.SetLastError(e2) set_errno(e) after_external_call._gctransformer_hint_cannot_collect_ = True after_external_call._dont_reach_me_in_del_ = True diff --git a/pypy/objspace/std/bufferobject.py b/pypy/objspace/std/bufferobject.py index 8f904bcd58..7c318ed912 100644 --- a/pypy/objspace/std/bufferobject.py +++ b/pypy/objspace/std/bufferobject.py @@ -5,7 +5,7 @@ from rpython.rlib.buffer import Buffer, SubBuffer from rpython.rlib.objectmodel import compute_hash from pypy.interpreter.baseobjspace import W_Root -from pypy.interpreter.error import oefmt +from pypy.interpreter.error import OperationError, oefmt from pypy.interpreter.gateway import interp2app, unwrap_spec from pypy.interpreter.typedef import TypeDef @@ -123,6 +123,17 @@ class W_Buffer(W_Root): return space.wrap("<%s for 0x%s, size %d>" % (info, addrstring, self.buf.getlength())) + def descr_pypy_raw_address(self, space): + from rpython.rtyper.lltypesystem import lltype, rffi + try: + ptr = self.buf.get_raw_address() + except ValueError: + # report the error using the RPython-level internal repr of self.buf + msg = ("cannot find the underlying address of buffer that " + "is internally %r" % (self.buf,)) + raise OperationError(space.w_ValueError, space.wrap(msg)) + return space.wrap(rffi.cast(lltype.Signed, ptr)) + W_Buffer.typedef = TypeDef( "buffer", __doc__ = """\ @@ -149,5 +160,6 @@ extend to the end of the target object (or with the specified size). __mul__ = interp2app(W_Buffer.descr_mul), __rmul__ = interp2app(W_Buffer.descr_mul), __repr__ = interp2app(W_Buffer.descr_repr), + _pypy_raw_address = interp2app(W_Buffer.descr_pypy_raw_address), ) W_Buffer.typedef.acceptable_as_base_class = False diff --git a/pypy/objspace/std/test/test_bufferobject.py b/pypy/objspace/std/test/test_bufferobject.py index c069bd571c..9c9611b561 100644 --- a/pypy/objspace/std/test/test_bufferobject.py +++ b/pypy/objspace/std/test/test_bufferobject.py @@ -197,3 +197,9 @@ class AppTestBuffer: buf = buffer('hello world') raises(TypeError, "buf[MyInt(0)]") raises(TypeError, "buf[MyInt(0):MyInt(5)]") + + def test_pypy_raw_address_base(self): + raises(ValueError, buffer("foobar")._pypy_raw_address) + raises(ValueError, buffer(u"foobar")._pypy_raw_address) + e = raises(ValueError, buffer(bytearray("foobar"))._pypy_raw_address) + assert 'BytearrayBuffer' in str(e.value) diff --git a/rpython/annotator/classdef.py b/rpython/annotator/classdef.py index fae08a0ce0..f8cdd9d79b 100644 --- a/rpython/annotator/classdef.py +++ b/rpython/annotator/classdef.py @@ -451,11 +451,3 @@ except NameError: pass else: FORCE_ATTRIBUTES_INTO_CLASSES[WindowsError] = {'winerror': SomeInteger()} - -try: - import termios -except ImportError: - pass -else: - FORCE_ATTRIBUTES_INTO_CLASSES[termios.error] = \ - {'args': SomeTuple([SomeInteger(), SomeString()])} diff --git a/rpython/annotator/model.py b/rpython/annotator/model.py index 278e0a080f..d2fb715d46 100644 --- a/rpython/annotator/model.py +++ b/rpython/annotator/model.py @@ -254,7 +254,10 @@ class SomeStringOrUnicode(SomeObject): return self.__class__(can_be_None=False, no_nul=self.no_nul) def nonnulify(self): - return self.__class__(can_be_None=self.can_be_None, no_nul=True) + if self.can_be_None: + return self.__class__(can_be_None=True, no_nul=True) + else: + return self.__class__(no_nul=True) class SomeString(SomeStringOrUnicode): diff --git a/rpython/annotator/test/test_annrpython.py b/rpython/annotator/test/test_annrpython.py index 29bfeae1d9..b60259ac9d 100644 --- a/rpython/annotator/test/test_annrpython.py +++ b/rpython/annotator/test/test_annrpython.py @@ -4326,6 +4326,13 @@ class TestAnnotateTestCase: assert isinstance(s, annmodel.SomeString) assert not s.can_be_none() + def test_nonnulify(self): + s = annmodel.SomeString(can_be_None=True).nonnulify() + assert s.can_be_None is True + assert s.no_nul is True + s = annmodel.SomeChar().nonnulify() + assert s.no_nul is True + def g(n): return [0, 1, 2, n] diff --git a/rpython/jit/backend/arm/assembler.py b/rpython/jit/backend/arm/assembler.py index 0c9512223b..00821e85e1 100644 --- a/rpython/jit/backend/arm/assembler.py +++ b/rpython/jit/backend/arm/assembler.py @@ -901,7 +901,7 @@ class AssemblerARM(ResOpAssembler): descr = tok.faildescr assert isinstance(descr, AbstractFailDescr) failure_recovery_pos = block_start + tok.pos_recovery_stub - descr._arm_failure_recovery_block = failure_recovery_pos + descr.adr_jump_offset = failure_recovery_pos relative_offset = tok.pos_recovery_stub - tok.offset guard_pos = block_start + tok.offset if not tok.is_guard_not_invalidated: @@ -968,11 +968,11 @@ class AssemblerARM(ResOpAssembler): def patch_trace(self, faildescr, looptoken, bridge_addr, regalloc): b = InstrBuilder(self.cpu.cpuinfo.arch_version) - patch_addr = faildescr._arm_failure_recovery_block + patch_addr = faildescr.adr_jump_offset assert patch_addr != 0 b.B(bridge_addr) b.copy_to_raw_memory(patch_addr) - faildescr._arm_failure_recovery_block = 0 + faildescr.adr_jump_offset = 0 # regalloc support def load(self, loc, value): diff --git a/rpython/jit/backend/llsupport/assembler.py b/rpython/jit/backend/llsupport/assembler.py index 0b5f48a9d5..36f85af506 100644 --- a/rpython/jit/backend/llsupport/assembler.py +++ b/rpython/jit/backend/llsupport/assembler.py @@ -192,8 +192,6 @@ class BaseAssembler(object): positions[i] = rffi.cast(rffi.USHORT, position) # write down the positions of locs guardtok.faildescr.rd_locs = positions - # we want the descr to keep alive - guardtok.faildescr.rd_loop_token = self.current_clt return fail_descr, target def call_assembler(self, op, guard_op, argloc, vloc, result_loc, tmploc): diff --git a/rpython/jit/backend/x86/assembler.py b/rpython/jit/backend/x86/assembler.py index a10f6281ba..74dd83e562 100644 --- a/rpython/jit/backend/x86/assembler.py +++ b/rpython/jit/backend/x86/assembler.py @@ -298,22 +298,14 @@ class Assembler386(BaseAssembler): if slowpathaddr == 0 or not self.cpu.propagate_exception_descr: return # no stack check (for tests, or non-translated) # - # make a "function" that is called immediately at the start of - # an assembler function. In particular, the stack looks like: - # - # | ... | <-- aligned to a multiple of 16 - # | retaddr of caller | - # | my own retaddr | <-- esp - # +---------------------+ - # + # make a regular function that is called from a point near the start + # of an assembler function (after it adjusts the stack and saves + # registers). mc = codebuf.MachineCodeBlockWrapper() # if IS_X86_64: - # on the x86_64, we have to save all the registers that may - # have been used to pass arguments. Note that we pass only - # one argument, that is the frame mc.MOV_rr(edi.value, esp.value) - mc.SUB_ri(esp.value, WORD) + mc.SUB_ri(esp.value, WORD) # alignment # if IS_X86_32: mc.SUB_ri(esp.value, 2*WORD) # alignment @@ -572,13 +564,13 @@ class Assembler386(BaseAssembler): def patch_pending_failure_recoveries(self, rawstart): # after we wrote the assembler to raw memory, set up - # tok.faildescr._x86_adr_jump_offset to contain the raw address of + # tok.faildescr.adr_jump_offset to contain the raw address of # the 4-byte target field in the JMP/Jcond instruction, and patch # the field in question to point (initially) to the recovery stub clt = self.current_clt for tok in self.pending_guard_tokens: addr = rawstart + tok.pos_jump_offset - tok.faildescr._x86_adr_jump_offset = addr + tok.faildescr.adr_jump_offset = addr relative_target = tok.pos_recovery_stub - (tok.pos_jump_offset + 4) assert rx86.fits_in_32bits(relative_target) # @@ -685,7 +677,7 @@ class Assembler386(BaseAssembler): self.cpu.gc_ll_descr.gcrootmap) def patch_jump_for_descr(self, faildescr, adr_new_target): - adr_jump_offset = faildescr._x86_adr_jump_offset + adr_jump_offset = faildescr.adr_jump_offset assert adr_jump_offset != 0 offset = adr_new_target - (adr_jump_offset + 4) # If the new target fits within a rel32 of the jump, just patch @@ -705,7 +697,7 @@ class Assembler386(BaseAssembler): p = rffi.cast(rffi.INTP, adr_jump_offset) adr_target = adr_jump_offset + 4 + rffi.cast(lltype.Signed, p[0]) mc.copy_to_raw_memory(adr_target) - faildescr._x86_adr_jump_offset = 0 # means "patched" + faildescr.adr_jump_offset = 0 # means "patched" def fixup_target_tokens(self, rawstart): for targettoken in self.target_tokens_currently_compiling: diff --git a/rpython/jit/metainterp/blackhole.py b/rpython/jit/metainterp/blackhole.py index c9fd6a4f34..d965708fcc 100644 --- a/rpython/jit/metainterp/blackhole.py +++ b/rpython/jit/metainterp/blackhole.py @@ -1479,15 +1479,15 @@ class BlackholeInterpreter(object): assert kind == 'v' return lltype.nullptr(rclass.OBJECTPTR.TO) - def _prepare_resume_from_failure(self, opnum, dont_change_position, - deadframe): + def _prepare_resume_from_failure(self, opnum, deadframe): from rpython.jit.metainterp.resoperation import rop # - if opnum == rop.GUARD_TRUE: + if opnum == rop.GUARD_FUTURE_CONDITION: + pass + elif opnum == rop.GUARD_TRUE: # Produced directly by some goto_if_not_xxx() opcode that did not # jump, but which must now jump. The pc is just after the opcode. - if not dont_change_position: - self.position = self.jitcode.follow_jump(self.position) + self.position = self.jitcode.follow_jump(self.position) # elif opnum == rop.GUARD_FALSE: # Produced directly by some goto_if_not_xxx() opcode that jumped, @@ -1517,8 +1517,7 @@ class BlackholeInterpreter(object): elif opnum == rop.GUARD_NO_OVERFLOW: # Produced by int_xxx_ovf(). The pc is just after the opcode. # We get here because it did not used to overflow, but now it does. - if not dont_change_position: - return get_llexception(self.cpu, OverflowError()) + return get_llexception(self.cpu, OverflowError()) # elif opnum == rop.GUARD_OVERFLOW: # Produced by int_xxx_ovf(). The pc is just after the opcode. @@ -1649,13 +1648,9 @@ def resume_in_blackhole(metainterp_sd, jitdriver_sd, resumedescr, deadframe, resumedescr, deadframe, all_virtuals) - if isinstance(resumedescr, ResumeAtPositionDescr): - dont_change_position = True - else: - dont_change_position = False current_exc = blackholeinterp._prepare_resume_from_failure( - resumedescr.guard_opnum, dont_change_position, deadframe) + resumedescr.guard_opnum, deadframe) _run_forever(blackholeinterp, current_exc) diff --git a/rpython/jit/metainterp/compile.py b/rpython/jit/metainterp/compile.py index 214c044582..2f290614f3 100644 --- a/rpython/jit/metainterp/compile.py +++ b/rpython/jit/metainterp/compile.py @@ -63,14 +63,18 @@ def record_loop_or_bridge(metainterp_sd, loop): if metainterp_sd.warmrunnerdesc is not None: # for tests assert original_jitcell_token.generation > 0 # has been registered with memmgr wref = weakref.ref(original_jitcell_token) + clt = original_jitcell_token.compiled_loop_token + clt.loop_token_wref = wref for op in loop.operations: descr = op.getdescr() + # not sure what descr.index is about if isinstance(descr, ResumeDescr): - descr.wref_original_loop_token = wref # stick it there - n = descr.index - if n >= 0: # we also record the resumedescr number - original_jitcell_token.compiled_loop_token.record_faildescr_index(n) - elif isinstance(descr, JitCellToken): + descr.rd_loop_token = clt # stick it there + #n = descr.index + #if n >= 0: # we also record the resumedescr number + # original_jitcell_token.compiled_loop_token.record_faildescr_index(n) + # pass + if isinstance(descr, JitCellToken): # for a CALL_ASSEMBLER: record it as a potential jump. if descr is not original_jitcell_token: original_jitcell_token.record_jump_to(descr) @@ -123,13 +127,14 @@ def compile_loop(metainterp, greenkey, start, part = create_empty_loop(metainterp) part.inputargs = inputargs[:] h_ops = history.operations - part.operations = [ResOperation(rop.LABEL, inputargs, None, descr=TargetToken(jitcell_token))] + \ - [h_ops[i].clone() for i in range(start, len(h_ops))] + \ - [ResOperation(rop.LABEL, jumpargs, None, descr=jitcell_token)] + label = ResOperation(rop.LABEL, inputargs, None, + descr=TargetToken(jitcell_token)) + end_label = ResOperation(rop.LABEL, jumpargs, None, descr=jitcell_token) + part.operations = [label] + h_ops[start:] + [end_label] try: - start_state = optimize_trace(metainterp_sd, part, enable_opts, - export_state=True) + start_state = optimize_trace(metainterp_sd, jitdriver_sd, part, + enable_opts, export_state=True) except InvalidLoop: return None target_token = part.operations[0].getdescr() @@ -156,7 +161,7 @@ def compile_loop(metainterp, greenkey, start, jumpargs = part.operations[-1].getarglist() try: - optimize_trace(metainterp_sd, part, enable_opts, + optimize_trace(metainterp_sd, jitdriver_sd, part, enable_opts, start_state=start_state, export_state=False) except InvalidLoop: return None @@ -203,13 +208,14 @@ def compile_retrace(metainterp, greenkey, start, h_ops = history.operations part.operations = [partial_trace.operations[-1]] + \ - [h_ops[i].clone() for i in range(start, len(h_ops))] + \ + h_ops[start:] + \ [ResOperation(rop.JUMP, jumpargs, None, descr=loop_jitcell_token)] label = part.operations[0] orignial_label = label.clone() assert label.getopnum() == rop.LABEL try: - optimize_trace(metainterp_sd, part, jitdriver_sd.warmstate.enable_opts, + optimize_trace(metainterp_sd, jitdriver_sd, part, + jitdriver_sd.warmstate.enable_opts, start_state=start_state, export_state=False) except InvalidLoop: # Fall back on jumping to preamble @@ -219,7 +225,7 @@ def compile_retrace(metainterp, greenkey, start, [ResOperation(rop.JUMP, inputargs[:], None, descr=loop_jitcell_token)] try: - optimize_trace(metainterp_sd, part, + optimize_trace(metainterp_sd, jitdriver_sd, part, jitdriver_sd.warmstate.enable_opts, inline_short_preamble=False, start_state=start_state, export_state=False) @@ -479,22 +485,31 @@ def make_done_loop_tokens(): return d class ResumeDescr(AbstractFailDescr): - pass + _attrs_ = () class ResumeGuardDescr(ResumeDescr): - # this class also gets the following attributes stored by resume.py code - # XXX move all of unused stuff to guard_op, now that we can have - # a separate class, so it does not survive that long - rd_snapshot = None - rd_frame_info_list = None + _attrs_ = ('rd_numb', 'rd_count', 'rd_consts', 'rd_virtuals', + 'rd_frame_info_list', 'rd_pendingfields', 'status') + rd_numb = lltype.nullptr(NUMBERING) rd_count = 0 rd_consts = None rd_virtuals = None + rd_frame_info_list = None rd_pendingfields = lltype.nullptr(PENDINGFIELDSP.TO) status = r_uint(0) + def copy_all_attributes_from(self, other): + assert isinstance(other, ResumeGuardDescr) + self.rd_count = other.rd_count + self.rd_consts = other.rd_consts + self.rd_frame_info_list = other.rd_frame_info_list + self.rd_pendingfields = other.rd_pendingfields + self.rd_virtuals = other.rd_virtuals + self.rd_numb = other.rd_numb + # we don't copy status + ST_BUSY_FLAG = 0x01 # if set, busy tracing from the guard ST_TYPE_MASK = 0x06 # mask for the type (TY_xxx) ST_SHIFT = 3 # in "status >> ST_SHIFT" is stored: @@ -509,31 +524,12 @@ class ResumeGuardDescr(ResumeDescr): def store_final_boxes(self, guard_op, boxes, metainterp_sd): guard_op.setfailargs(boxes) self.rd_count = len(boxes) - self.guard_opnum = guard_op.getopnum() # if metainterp_sd.warmrunnerdesc is not None: # for tests jitcounter = metainterp_sd.warmrunnerdesc.jitcounter hash = jitcounter.fetch_next_hash() self.status = hash & self.ST_SHIFT_MASK - def make_a_counter_per_value(self, guard_value_op): - assert guard_value_op.getopnum() == rop.GUARD_VALUE - box = guard_value_op.getarg(0) - try: - i = guard_value_op.getfailargs().index(box) - except ValueError: - return # xxx probably very rare - else: - if box.type == history.INT: - ty = self.TY_INT - elif box.type == history.REF: - ty = self.TY_REF - elif box.type == history.FLOAT: - ty = self.TY_FLOAT - else: - assert 0, box.type - self.status = ty | (r_uint(i) << self.ST_SHIFT) - def handle_fail(self, deadframe, metainterp_sd, jitdriver_sd): if self.must_compile(deadframe, metainterp_sd, jitdriver_sd): self.start_compiling() @@ -632,32 +628,62 @@ class ResumeGuardDescr(ResumeDescr): self, inputargs, new_loop.operations, new_loop.original_jitcell_token) - def copy_all_attributes_into(self, res): - # XXX a bit ugly to have to list them all here - res.rd_snapshot = self.rd_snapshot - res.rd_frame_info_list = self.rd_frame_info_list - res.rd_numb = self.rd_numb - res.rd_consts = self.rd_consts - res.rd_virtuals = self.rd_virtuals - res.rd_pendingfields = self.rd_pendingfields - res.rd_count = self.rd_count - - def _clone_if_mutable(self): - res = ResumeGuardDescr() - self.copy_all_attributes_into(res) - return res + def make_a_counter_per_value(self, guard_value_op): + assert guard_value_op.getopnum() == rop.GUARD_VALUE + box = guard_value_op.getarg(0) + try: + i = guard_value_op.getfailargs().index(box) + except ValueError: + return # xxx probably very rare + else: + if box.type == history.INT: + ty = self.TY_INT + elif box.type == history.REF: + ty = self.TY_REF + elif box.type == history.FLOAT: + ty = self.TY_FLOAT + else: + assert 0, box.type + self.status = ty | (r_uint(i) << self.ST_SHIFT) + +class ResumeGuardNonnullDescr(ResumeGuardDescr): + guard_opnum = rop.GUARD_NONNULL + +class ResumeGuardIsnullDescr(ResumeGuardDescr): + guard_opnum = rop.GUARD_ISNULL + +class ResumeGuardClassDescr(ResumeGuardDescr): + guard_opnum = rop.GUARD_CLASS + +class ResumeGuardTrueDescr(ResumeGuardDescr): + guard_opnum = rop.GUARD_TRUE + +class ResumeGuardFalseDescr(ResumeGuardDescr): + guard_opnum = rop.GUARD_FALSE + +class ResumeGuardNonnullClassDescr(ResumeGuardDescr): + guard_opnum = rop.GUARD_NONNULL_CLASS + +class ResumeGuardExceptionDescr(ResumeGuardDescr): + guard_opnum = rop.GUARD_EXCEPTION + +class ResumeGuardNoExceptionDescr(ResumeGuardDescr): + guard_opnum = rop.GUARD_NO_EXCEPTION + +class ResumeGuardOverflowDescr(ResumeGuardDescr): + guard_opnum = rop.GUARD_OVERFLOW + +class ResumeGuardNoOverflowDescr(ResumeGuardDescr): + guard_opnum = rop.GUARD_NO_OVERFLOW + +class ResumeGuardValueDescr(ResumeGuardDescr): + guard_opnum = rop.GUARD_VALUE class ResumeGuardNotInvalidated(ResumeGuardDescr): - def _clone_if_mutable(self): - res = ResumeGuardNotInvalidated() - self.copy_all_attributes_into(res) - return res + guard_opnum = rop.GUARD_NOT_INVALIDATED class ResumeAtPositionDescr(ResumeGuardDescr): - def _clone_if_mutable(self): - res = ResumeAtPositionDescr() - self.copy_all_attributes_into(res) - return res + guard_opnum = rop.GUARD_FUTURE_CONDITION class AllVirtuals: llopaque = True @@ -678,8 +704,10 @@ class AllVirtuals: class ResumeGuardForcedDescr(ResumeGuardDescr): + guard_opnum = rop.GUARD_NOT_FORCED - def __init__(self, metainterp_sd, jitdriver_sd): + def _init(self, metainterp_sd, jitdriver_sd): + # to please the annotator self.metainterp_sd = metainterp_sd self.jitdriver_sd = jitdriver_sd @@ -740,12 +768,39 @@ class ResumeGuardForcedDescr(ResumeGuardDescr): hidden_all_virtuals = obj.hide(metainterp_sd.cpu) metainterp_sd.cpu.set_savedata_ref(deadframe, hidden_all_virtuals) - def _clone_if_mutable(self): - res = ResumeGuardForcedDescr(self.metainterp_sd, - self.jitdriver_sd) - self.copy_all_attributes_into(res) - return res - +def invent_fail_descr_for_op(opnum, optimizer): + if opnum == rop.GUARD_NOT_FORCED or opnum == rop.GUARD_NOT_FORCED_2: + resumedescr = ResumeGuardForcedDescr() + resumedescr._init(optimizer.metainterp_sd, optimizer.jitdriver_sd) + elif opnum == rop.GUARD_NOT_INVALIDATED: + resumedescr = ResumeGuardNotInvalidated() + elif opnum == rop.GUARD_FUTURE_CONDITION: + resumedescr = ResumeAtPositionDescr() + elif opnum == rop.GUARD_VALUE: + resumedescr = ResumeGuardValueDescr() + elif opnum == rop.GUARD_NONNULL: + resumedescr = ResumeGuardNonnullDescr() + elif opnum == rop.GUARD_ISNULL: + resumedescr = ResumeGuardIsnullDescr() + elif opnum == rop.GUARD_NONNULL_CLASS: + resumedescr = ResumeGuardNonnullClassDescr() + elif opnum == rop.GUARD_CLASS: + resumedescr = ResumeGuardClassDescr() + elif opnum == rop.GUARD_TRUE: + resumedescr = ResumeGuardTrueDescr() + elif opnum == rop.GUARD_FALSE: + resumedescr = ResumeGuardFalseDescr() + elif opnum == rop.GUARD_EXCEPTION: + resumedescr = ResumeGuardExceptionDescr() + elif opnum == rop.GUARD_NO_EXCEPTION: + resumedescr = ResumeGuardNoExceptionDescr() + elif opnum == rop.GUARD_OVERFLOW: + resumedescr = ResumeGuardOverflowDescr() + elif opnum == rop.GUARD_NO_OVERFLOW: + resumedescr = ResumeGuardNoOverflowDescr() + else: + assert False + return resumedescr class ResumeFromInterpDescr(ResumeDescr): def __init__(self, original_greenkey): @@ -781,17 +836,18 @@ def compile_trace(metainterp, resumekey): # it does not work -- i.e. none of the existing old_loop_tokens match. new_trace = create_empty_loop(metainterp) new_trace.inputargs = metainterp.history.inputargs[:] - # clone ops, as optimize_bridge can mutate the ops - new_trace.operations = [op.clone() for op in metainterp.history.operations] + new_trace.operations = metainterp.history.operations[:] metainterp_sd = metainterp.staticdata - state = metainterp.jitdriver_sd.warmstate + jitdriver_sd = metainterp.jitdriver_sd + state = jitdriver_sd.warmstate if isinstance(resumekey, ResumeAtPositionDescr): inline_short_preamble = False else: inline_short_preamble = True try: - state = optimize_trace(metainterp_sd, new_trace, state.enable_opts, + state = optimize_trace(metainterp_sd, jitdriver_sd, new_trace, + state.enable_opts, inline_short_preamble, export_state=True) except InvalidLoop: debug_print("compile_new_bridge: got an InvalidLoop") diff --git a/rpython/jit/metainterp/history.py b/rpython/jit/metainterp/history.py index 3705882b4e..4731e92e50 100644 --- a/rpython/jit/metainterp/history.py +++ b/rpython/jit/metainterp/history.py @@ -134,14 +134,6 @@ class AbstractDescr(AbstractValue): def repr_of_descr(self): return '%r' % (self,) - def _clone_if_mutable(self): - return self - def clone_if_mutable(self): - clone = self._clone_if_mutable() - if not we_are_translated(): - assert clone.__class__ is self.__class__ - return clone - def hide(self, cpu): descr_ptr = cpu.ts.cast_instance_to_base_ref(self) return cpu.ts.cast_to_ref(descr_ptr) @@ -159,6 +151,8 @@ class AbstractFailDescr(AbstractDescr): index = -1 final_descr = False + _attrs_ = ('adr_jump_offset', 'rd_locs', 'rd_loop_token') + def handle_fail(self, deadframe, metainterp_sd, jitdriver_sd): raise NotImplementedError def compile_and_attach(self, metainterp, new_loop): diff --git a/rpython/jit/metainterp/inliner.py b/rpython/jit/metainterp/inliner.py index 201628e100..e8bc27998d 100644 --- a/rpython/jit/metainterp/inliner.py +++ b/rpython/jit/metainterp/inliner.py @@ -1,5 +1,6 @@ from rpython.jit.metainterp.history import Const from rpython.jit.metainterp.resume import Snapshot +from rpython.jit.metainterp.resoperation import GuardResOp class Inliner(object): @@ -26,21 +27,16 @@ class Inliner(object): newop.setfailargs([self.inline_arg(a) for a in args]) else: newop.setfailargs([]) + assert isinstance(newop, GuardResOp) + newop.rd_snapshot = self.inline_snapshot(newop.rd_snapshot) if newop.result and not ignore_result: old_result = newop.result newop.result = newop.result.clonebox() self.argmap[old_result] = newop.result - self.inline_descr_inplace(newop.getdescr()) - return newop - def inline_descr_inplace(self, descr): - from rpython.jit.metainterp.compile import ResumeGuardDescr - if isinstance(descr, ResumeGuardDescr): - descr.rd_snapshot = self.inline_snapshot(descr.rd_snapshot) - def inline_arg(self, arg): if arg is None: return None diff --git a/rpython/jit/metainterp/optimizeopt/__init__.py b/rpython/jit/metainterp/optimizeopt/__init__.py index eb327edc07..804f4d7c1c 100644 --- a/rpython/jit/metainterp/optimizeopt/__init__.py +++ b/rpython/jit/metainterp/optimizeopt/__init__.py @@ -47,7 +47,7 @@ def build_opt_chain(metainterp_sd, enable_opts): return optimizations, unroll -def optimize_trace(metainterp_sd, loop, enable_opts, +def optimize_trace(metainterp_sd, jitdriver_sd, loop, enable_opts, inline_short_preamble=True, start_state=None, export_state=True): """Optimize loop.operations to remove internal overheadish operations. @@ -59,11 +59,13 @@ def optimize_trace(metainterp_sd, loop, enable_opts, loop.operations) optimizations, unroll = build_opt_chain(metainterp_sd, enable_opts) if unroll: - return optimize_unroll(metainterp_sd, loop, optimizations, + return optimize_unroll(metainterp_sd, jitdriver_sd, loop, + optimizations, inline_short_preamble, start_state, export_state) else: - optimizer = Optimizer(metainterp_sd, loop, optimizations) + optimizer = Optimizer(metainterp_sd, jitdriver_sd, loop, + optimizations) optimizer.propagate_all_forward() finally: debug_stop("jit-optimize") diff --git a/rpython/jit/metainterp/optimizeopt/heap.py b/rpython/jit/metainterp/optimizeopt/heap.py index 930eebcc5b..258c80d55c 100644 --- a/rpython/jit/metainterp/optimizeopt/heap.py +++ b/rpython/jit/metainterp/optimizeopt/heap.py @@ -4,7 +4,8 @@ from rpython.jit.codewriter.effectinfo import EffectInfo from rpython.jit.metainterp.optimizeopt.util import args_dict from rpython.jit.metainterp.history import Const, ConstInt from rpython.jit.metainterp.jitexc import JitException -from rpython.jit.metainterp.optimizeopt.optimizer import Optimization, MODE_ARRAY, LEVEL_KNOWNCLASS, REMOVED, LEVEL_CONSTANT +from rpython.jit.metainterp.optimizeopt.optimizer import Optimization,\ + MODE_ARRAY, LEVEL_KNOWNCLASS, REMOVED from rpython.jit.metainterp.optimizeopt.util import make_dispatcher_method from rpython.jit.metainterp.optimizeopt.intutils import IntBound from rpython.jit.metainterp.optimize import InvalidLoop @@ -64,16 +65,17 @@ class CachedField(object): # cancelling its previous effects with no side effect. self._lazy_setfield = None - def value_updated(self, oldvalue, newvalue): + def value_updated(self, oldvalue, newvalue, exporting_state): try: fieldvalue = self._cached_fields[oldvalue] except KeyError: pass else: self._cached_fields[newvalue] = fieldvalue - op = self._cached_fields_getfield_op[oldvalue].clone() - op.setarg(0, newvalue.box) - self._cached_fields_getfield_op[newvalue] = op + if exporting_state: + op = self._cached_fields_getfield_op[oldvalue].clone() + op.setarg(0, newvalue.box) + self._cached_fields_getfield_op[newvalue] = op def possible_aliasing(self, optheap, structvalue): # If lazy_setfield is set and contains a setfield on a different @@ -94,10 +96,13 @@ class CachedField(object): else: return self._cached_fields.get(structvalue, None) - def remember_field_value(self, structvalue, fieldvalue, getfield_op=None): + def remember_field_value(self, structvalue, fieldvalue, op=None, + optimizer=None): assert self._lazy_setfield is None self._cached_fields[structvalue] = fieldvalue - self._cached_fields_getfield_op[structvalue] = getfield_op + if optimizer.exporting_state: + op = optimizer.get_op_replacement(op) + self._cached_fields_getfield_op[structvalue] = op def force_lazy_setfield(self, optheap, can_cache=True): op = self._lazy_setfield @@ -121,7 +126,8 @@ class CachedField(object): # field. structvalue = optheap.getvalue(op.getarg(0)) fieldvalue = optheap.getvalue(op.getarglist()[-1]) - self.remember_field_value(structvalue, fieldvalue, op) + self.remember_field_value(structvalue, fieldvalue, op, + optheap.optimizer) elif not can_cache: self.clear() @@ -130,6 +136,7 @@ class CachedField(object): self._cached_fields_getfield_op.clear() def produce_potential_short_preamble_ops(self, optimizer, shortboxes, descr): + assert optimizer.exporting_state if self._lazy_setfield is not None: return for structvalue in self._cached_fields_getfield_op.keys(): @@ -138,7 +145,7 @@ class CachedField(object): continue value = optimizer.getvalue(op.getarg(0)) if value in optimizer.opaque_pointers: - if value.level < LEVEL_KNOWNCLASS: + if value.getlevel() < LEVEL_KNOWNCLASS: continue if op.getopnum() != rop.SETFIELD_GC and op.getopnum() != rop.GETFIELD_GC: continue @@ -192,10 +199,11 @@ class OptHeap(Optimization): def value_updated(self, oldvalue, newvalue): # XXXX very unhappy about that for cf in self.cached_fields.itervalues(): - cf.value_updated(oldvalue, newvalue) + cf.value_updated(oldvalue, newvalue, self.optimizer.exporting_state) for submap in self.cached_arrayitems.itervalues(): for cf in submap.itervalues(): - cf.value_updated(oldvalue, newvalue) + cf.value_updated(oldvalue, newvalue, + self.optimizer.exporting_state) def force_at_end_of_preamble(self): self.cached_dict_reads.clear() @@ -350,8 +358,8 @@ class OptHeap(Optimization): d = self.cached_dict_reads[descr1] = args_dict() self.corresponding_array_descrs[descrs[1]] = descr1 # - key = [self.optimizer.get_arg_key(op.getarg(1)), # dict - self.optimizer.get_arg_key(op.getarg(2))] # key + key = [self.optimizer.get_box_replacement(op.getarg(1)), # dict + self.optimizer.get_box_replacement(op.getarg(2))] # key # other args can be ignored here (hash, store_flag) try: res_v = d[key] @@ -476,7 +484,7 @@ class OptHeap(Optimization): self.emit_operation(op) # then remember the result of reading the field fieldvalue = self.getvalue(op.result) - cf.remember_field_value(structvalue, fieldvalue, op) + cf.remember_field_value(structvalue, fieldvalue, op, self.optimizer) def optimize_GETFIELD_GC_PURE(self, op): structvalue = self.getvalue(op.getarg(0)) @@ -520,7 +528,7 @@ class OptHeap(Optimization): # the remember the result of reading the array item if cf is not None: fieldvalue = self.getvalue(op.result) - cf.remember_field_value(arrayvalue, fieldvalue, op) + cf.remember_field_value(arrayvalue, fieldvalue, op, self.optimizer) def optimize_GETARRAYITEM_GC_PURE(self, op): arrayvalue = self.getvalue(op.getarg(0)) diff --git a/rpython/jit/metainterp/optimizeopt/optimizer.py b/rpython/jit/metainterp/optimizeopt/optimizer.py index 680bc989d4..a8901f3167 100644 --- a/rpython/jit/metainterp/optimizeopt/optimizer.py +++ b/rpython/jit/metainterp/optimizeopt/optimizer.py @@ -7,17 +7,23 @@ from rpython.jit.metainterp.optimizeopt.intutils import IntBound, IntUnbounded,\ IntLowerBound, MININT,\ MAXINT from rpython.jit.metainterp.optimizeopt.util import make_dispatcher_method -from rpython.jit.metainterp.resoperation import rop, ResOperation, AbstractResOp +from rpython.jit.metainterp.resoperation import rop, ResOperation,\ + AbstractResOp, GuardResOp from rpython.jit.metainterp.typesystem import llhelper from rpython.tool.pairtype import extendabletype from rpython.rlib.debug import debug_print from rpython.rlib.objectmodel import specialize +""" The tag field on OptValue has a following meaning: -LEVEL_UNKNOWN = '\x00' -LEVEL_NONNULL = '\x01' -LEVEL_KNOWNCLASS = '\x02' # might also mean KNOWNARRAYDESCR, for arrays -LEVEL_CONSTANT = '\x03' +lower two bits are LEVEL +next 16 bits is the position in the original list, 0 if unknown or a constant +""" + +LEVEL_UNKNOWN = 0 +LEVEL_NONNULL = 1 +LEVEL_KNOWNCLASS = 2 # might also mean KNOWNARRAYDESCR, for arrays +LEVEL_CONSTANT = 3 MODE_ARRAY = '\x00' MODE_STR = '\x01' @@ -41,35 +47,41 @@ class LenBound(object): class OptValue(object): __metaclass__ = extendabletype - _attrs_ = ('box', 'level') + _attrs_ = ('box', '_tag') - level = LEVEL_UNKNOWN + _tag = 0 def __init__(self, box, level=None, known_class=None, intbound=None): self.box = box if level is not None: - self.level = level + self._tag = level if isinstance(box, Const): self.make_constant(box) # invariant: box is a Const if and only if level == LEVEL_CONSTANT + def getlevel(self): + return self._tag & 0x3 + + def setlevel(self, level): + self._tag = (self._tag & (~0x3)) | level + def import_from(self, other, optimizer): - if self.level == LEVEL_CONSTANT: - assert other.level == LEVEL_CONSTANT + if self.getlevel() == LEVEL_CONSTANT: + assert other.getlevel() == LEVEL_CONSTANT assert other.box.same_constant(self.box) return - assert self.level <= LEVEL_NONNULL - if other.level == LEVEL_CONSTANT: + assert self.getlevel() <= LEVEL_NONNULL + if other.getlevel() == LEVEL_CONSTANT: self.make_constant(other.get_key_box()) - elif other.level == LEVEL_KNOWNCLASS: - self.make_constant_class(other.get_known_class(), None) + elif other.getlevel() == LEVEL_KNOWNCLASS: + self.make_constant_class(None, other.get_known_class()) else: - if other.level == LEVEL_NONNULL: + if other.getlevel() == LEVEL_NONNULL: self.ensure_nonnull() def make_guards(self, box): - if self.level == LEVEL_CONSTANT: + if self.getlevel() == LEVEL_CONSTANT: op = ResOperation(rop.GUARD_VALUE, [box, self.box], None) return [op] return [] @@ -77,7 +89,7 @@ class OptValue(object): def copy_from(self, other_value): assert isinstance(other_value, OptValue) self.box = other_value.box - self.level = other_value.level + self._tag = other_value._tag def force_box(self, optforce): return self.box @@ -105,7 +117,7 @@ class OptValue(object): assert 0, "unreachable" def is_constant(self): - return self.level == LEVEL_CONSTANT + return self.getlevel() == LEVEL_CONSTANT def is_null(self): if self.is_constant(): @@ -122,7 +134,7 @@ class OptValue(object): return self is other def is_nonnull(self): - level = self.level + level = self.getlevel() if level == LEVEL_NONNULL or level == LEVEL_KNOWNCLASS: return True elif level == LEVEL_CONSTANT: @@ -133,8 +145,8 @@ class OptValue(object): return False def ensure_nonnull(self): - if self.level < LEVEL_NONNULL: - self.level = LEVEL_NONNULL + if self.getlevel() < LEVEL_NONNULL: + self.setlevel(LEVEL_NONNULL) def get_constant_int(self): assert self.is_constant() @@ -185,9 +197,9 @@ class OptValue(object): """Replace 'self.box' with a Const box.""" assert isinstance(constbox, Const) self.box = constbox - self.level = LEVEL_CONSTANT + self.setlevel(LEVEL_CONSTANT) - def get_last_guard(self): + def get_last_guard(self, optimizer): return None def get_known_class(self): @@ -203,10 +215,10 @@ class OptValue(object): return None class PtrOptValue(OptValue): - _attrs_ = ('known_class', 'last_guard', 'lenbound') + _attrs_ = ('known_class', 'last_guard_pos', 'lenbound') known_class = None - last_guard = None + last_guard_pos = -1 lenbound = None def __init__(self, box, level=None, known_class=None, intbound=None): @@ -218,32 +230,39 @@ class PtrOptValue(OptValue): assert isinstance(other_value, PtrOptValue) self.box = other_value.box self.known_class = other_value.known_class - self.level = other_value.level - self.last_guard = other_value.last_guard + self._tag = other_value._tag + self.last_guard_pos = other_value.last_guard_pos self.lenbound = other_value.lenbound def make_len_gt(self, mode, descr, val): if self.lenbound: - assert self.lenbound.mode == mode - assert self.lenbound.descr == descr + if self.lenbound.mode != mode or self.lenbound.descr != descr: + # XXX a rare case? it seems to occur sometimes when + # running lib-python's test_io.py in PyPy on Linux 32... + from rpython.jit.metainterp.optimize import InvalidLoop + raise InvalidLoop("bad mode/descr") self.lenbound.bound.make_gt(IntBound(val, val)) else: self.lenbound = LenBound(mode, descr, IntLowerBound(val + 1)) - def make_nonnull(self, guardop): - assert self.level < LEVEL_NONNULL - self.level = LEVEL_NONNULL - self.last_guard = guardop + def make_nonnull(self, optimizer): + assert self.getlevel() < LEVEL_NONNULL + self.setlevel(LEVEL_NONNULL) + if optimizer is not None: + self.last_guard_pos = len(optimizer._newoperations) - 1 + assert self.get_last_guard(optimizer).is_guard() - def make_constant_class(self, classbox, guardop): - assert self.level < LEVEL_KNOWNCLASS + def make_constant_class(self, optimizer, classbox): + assert self.getlevel() < LEVEL_KNOWNCLASS self.known_class = classbox - self.level = LEVEL_KNOWNCLASS - self.last_guard = guardop + self.setlevel(LEVEL_KNOWNCLASS) + if optimizer is not None: + self.last_guard_pos = len(optimizer._newoperations) - 1 + assert self.get_last_guard(optimizer).is_guard() def import_from(self, other, optimizer): OptValue.import_from(self, other, optimizer) - if self.level != LEVEL_CONSTANT: + if self.getlevel() != LEVEL_CONSTANT: if other.getlenbound(): if self.lenbound: assert other.getlenbound().mode == self.lenbound.mode @@ -254,16 +273,16 @@ class PtrOptValue(OptValue): def make_guards(self, box): guards = [] - if self.level == LEVEL_CONSTANT: + level = self.getlevel() + if level == LEVEL_CONSTANT: op = ResOperation(rop.GUARD_VALUE, [box, self.box], None) guards.append(op) - elif self.level == LEVEL_KNOWNCLASS: - op = ResOperation(rop.GUARD_NONNULL, [box], None) - guards.append(op) - op = ResOperation(rop.GUARD_CLASS, [box, self.known_class], None) + elif level == LEVEL_KNOWNCLASS: + op = ResOperation(rop.GUARD_NONNULL_CLASS, + [box, self.known_class], None) guards.append(op) else: - if self.level == LEVEL_NONNULL: + if level == LEVEL_NONNULL: op = ResOperation(rop.GUARD_NONNULL, [box], None) guards.append(op) if self.lenbound: @@ -282,7 +301,7 @@ class PtrOptValue(OptValue): return guards def get_constant_class(self, cpu): - level = self.level + level = self.getlevel() if level == LEVEL_KNOWNCLASS: return self.known_class elif level == LEVEL_CONSTANT: @@ -293,8 +312,10 @@ class PtrOptValue(OptValue): def getlenbound(self): return self.lenbound - def get_last_guard(self): - return self.last_guard + def get_last_guard(self, optimizer): + if self.last_guard_pos == -1: + return None + return optimizer._newoperations[self.last_guard_pos] def get_known_class(self): return self.known_class @@ -320,13 +341,13 @@ class IntOptValue(OptValue): assert isinstance(other_value, IntOptValue) self.box = other_value.box self.intbound = other_value.intbound - self.level = other_value.level + self._tag = other_value._tag def make_constant(self, constbox): """Replace 'self.box' with a Const box.""" assert isinstance(constbox, ConstInt) self.box = constbox - self.level = LEVEL_CONSTANT + self.setlevel(LEVEL_CONSTANT) val = constbox.getint() self.intbound = IntBound(val, val) @@ -339,26 +360,27 @@ class IntOptValue(OptValue): return True return False - def make_nonnull(self, guardop): - assert self.level < LEVEL_NONNULL - self.level = LEVEL_NONNULL + def make_nonnull(self, optimizer): + assert self.getlevel() < LEVEL_NONNULL + self.setlevel(LEVEL_NONNULL) def import_from(self, other, optimizer): OptValue.import_from(self, other, optimizer) - if self.level != LEVEL_CONSTANT: + if self.getlevel() != LEVEL_CONSTANT: if other.getintbound() is not None: # VRawBufferValue self.intbound.intersect(other.getintbound()) def make_guards(self, box): guards = [] - if self.level == LEVEL_CONSTANT: + level = self.getlevel() + if level == LEVEL_CONSTANT: op = ResOperation(rop.GUARD_VALUE, [box, self.box], None) guards.append(op) - elif self.level == LEVEL_KNOWNCLASS: + elif level == LEVEL_KNOWNCLASS: op = ResOperation(rop.GUARD_NONNULL, [box], None) guards.append(op) else: - if self.level == LEVEL_NONNULL: + if level == LEVEL_NONNULL: op = ResOperation(rop.GUARD_NONNULL, [box], None) guards.append(op) self.intbound.make_guards(box, guards) @@ -367,7 +389,7 @@ class IntOptValue(OptValue): def getintbound(self): return self.intbound - def get_last_guard(self): + def get_last_guard(self, optimizer): return None def get_known_class(self): @@ -494,8 +516,11 @@ class Optimization(object): class Optimizer(Optimization): - def __init__(self, metainterp_sd, loop, optimizations=None): + exporting_state = False + + def __init__(self, metainterp_sd, jitdriver_sd, loop, optimizations=None): self.metainterp_sd = metainterp_sd + self.jitdriver_sd = jitdriver_sd self.cpu = metainterp_sd.cpu self.loop = loop self.values = {} @@ -514,6 +539,8 @@ class Optimizer(Optimization): self.optpure = None self.optheap = None self.optearlyforce = None + # the following two fields is the data kept for unrolling, + # those are the operations that can go to the short_preamble if loop is not None: self.call_pure_results = loop.call_pure_results @@ -536,6 +563,12 @@ class Optimizer(Optimization): self.optimizations = optimizations + def replace_guard(self, op, value): + assert isinstance(value, PtrOptValue) + if value.last_guard_pos == -1: + return + self.replaces_guard[op] = value.last_guard_pos + def force_at_end_of_preamble(self): for o in self.optimizations: o.force_at_end_of_preamble() @@ -582,6 +615,13 @@ class Optimizer(Optimization): self.ensure_imported(value) return value + def get_box_replacement(self, box): + try: + v = self.values[box] + except KeyError: + return box + return v.get_key_box() + def ensure_imported(self, value): pass @@ -615,8 +655,7 @@ class Optimizer(Optimization): except KeyError: pass else: - assert value.level != LEVEL_CONSTANT - assert cur_value.level != LEVEL_CONSTANT + assert cur_value.getlevel() != LEVEL_CONSTANT # replacing with a different box cur_value.copy_from(value) return @@ -696,6 +735,8 @@ class Optimizer(Optimization): @specialize.argtype(0) def _emit_operation(self, op): assert op.getopnum() != rop.CALL_PURE + changed = False + orig_op = op for i in range(op.numargs()): arg = op.getarg(i) try: @@ -704,38 +745,65 @@ class Optimizer(Optimization): pass else: self.ensure_imported(value) - op.setarg(i, value.force_box(self)) + newbox = value.force_box(self) + if arg is not newbox: + if not changed: + op = op.clone() + changed = True + op.setarg(i, newbox) self.metainterp_sd.profiler.count(jitprof.Counters.OPT_OPS) if op.is_guard(): self.metainterp_sd.profiler.count(jitprof.Counters.OPT_GUARDS) pendingfields = self.pendingfields self.pendingfields = None - if self.replaces_guard and op in self.replaces_guard: - self.replace_op(self.replaces_guard[op], op) + if self.replaces_guard and orig_op in self.replaces_guard: + self.replace_op(self.replaces_guard[orig_op], op) del self.replaces_guard[op] return else: - op = self.store_final_boxes_in_guard(op, pendingfields) + guard_op = op.clone() + op = self.store_final_boxes_in_guard(guard_op, pendingfields) elif op.can_raise(): self.exception_might_have_happened = True + self._last_emitted_op = orig_op self._newoperations.append(op) - def replace_op(self, old_op, new_op): - # XXX: Do we want to cache indexes to prevent search? - i = len(self._newoperations) - while i > 0: - i -= 1 - if self._newoperations[i] is old_op: - self._newoperations[i] = new_op - break - else: - assert False + def get_op_replacement(self, op): + changed = False + for i, arg in enumerate(op.getarglist()): + try: + v = self.values[arg] + except KeyError: + pass + else: + box = v.get_key_box() + if box is not arg: + if not changed: + changed = True + op = op.clone() + op.setarg(i, box) + return op + + def replace_op(self, old_op_pos, new_op): + old_op = self._newoperations[old_op_pos] + assert old_op.is_guard() + old_descr = old_op.getdescr() + new_descr = new_op.getdescr() + new_descr.copy_all_attributes_from(old_descr) + self._newoperations[old_op_pos] = new_op def store_final_boxes_in_guard(self, op, pendingfields): assert pendingfields is not None - descr = op.getdescr() + if op.getdescr() is not None: + descr = op.getdescr() + assert isinstance(descr, compile.ResumeAtPositionDescr) + else: + descr = compile.invent_fail_descr_for_op(op.getopnum(), self) + op.setdescr(descr) assert isinstance(descr, compile.ResumeGuardDescr) - modifier = resume.ResumeDataVirtualAdder(descr, self.resumedata_memo) + assert isinstance(op, GuardResOp) + modifier = resume.ResumeDataVirtualAdder(descr, op, + self.resumedata_memo) try: newboxes = modifier.finish(self, pendingfields) if len(newboxes) > self.metainterp_sd.options.failargs_limit: @@ -765,20 +833,12 @@ class Optimizer(Optimization): descr.make_a_counter_per_value(op) return op - def get_arg_key(self, box): - try: - value = self.values[box] - except KeyError: - pass - else: - box = value.get_key_box() - return box - def make_args_key(self, op): n = op.numargs() args = [None] * (n + 2) for i in range(n): - args[i] = self.get_arg_key(op.getarg(i)) + arg = self.get_box_replacement(op.getarg(i)) + args[i] = arg args[n] = ConstInt(op.getopnum()) args[n + 1] = op.getdescr() return args @@ -793,6 +853,27 @@ class Optimizer(Optimization): op.getopnum(), argboxes, op.getdescr()) return resbox.constbox() + def pure_reverse(self, op): + if self.optpure is None: + return + optpure = self.optpure + if op.getopnum() == rop.INT_ADD: + optpure.pure(rop.INT_ADD, [op.getarg(1), op.getarg(0)], op.result) + # Synthesize the reverse op for optimize_default to reuse + optpure.pure(rop.INT_SUB, [op.result, op.getarg(1)], op.getarg(0)) + optpure.pure(rop.INT_SUB, [op.result, op.getarg(0)], op.getarg(1)) + elif op.getopnum() == rop.INT_SUB: + optpure.pure(rop.INT_ADD, [op.result, op.getarg(1)], op.getarg(0)) + optpure.pure(rop.INT_SUB, [op.getarg(0), op.result], op.getarg(1)) + elif op.getopnum() == rop.FLOAT_MUL: + optpure.pure(rop.FLOAT_MUL, [op.getarg(1), op.getarg(0)], op.result) + elif op.getopnum() == rop.FLOAT_NEG: + optpure.pure(rop.FLOAT_NEG, [op.result], op.getarg(0)) + elif op.getopnum() == rop.CAST_INT_TO_PTR: + optpure.pure(rop.CAST_PTR_TO_INT, [op.result], op.getarg(0)) + elif op.getopnum() == rop.CAST_PTR_TO_INT: + optpure.pure(rop.CAST_INT_TO_PTR, [op.result], op.getarg(0)) + #def optimize_GUARD_NO_OVERFLOW(self, op): # # otherwise the default optimizer will clear fields, which is unwanted # # in this case diff --git a/rpython/jit/metainterp/optimizeopt/pure.py b/rpython/jit/metainterp/optimizeopt/pure.py index 0b0999ad8f..e065252907 100644 --- a/rpython/jit/metainterp/optimizeopt/pure.py +++ b/rpython/jit/metainterp/optimizeopt/pure.py @@ -7,7 +7,7 @@ class OptPure(Optimization): def __init__(self): self.postponed_op = None self.pure_operations = args_dict() - self.emitted_pure_operations = {} + self.call_pure_positions = [] def propagate_forward(self, op): dispatch_opt(self, op) @@ -44,8 +44,6 @@ class OptPure(Optimization): if oldvalue is not None: self.optimizer.make_equal_to(op.result, oldvalue, True) return - else: - self.remember_emitting_pure(op) # otherwise, the operation remains self.emit_operation(op) @@ -77,12 +75,12 @@ class OptPure(Optimization): return else: self.pure_operations[args] = self.getvalue(op.result) - self.remember_emitting_pure(op) # replace CALL_PURE with just CALL args = op.getarglist() self.emit_operation(ResOperation(rop.CALL, args, op.result, op.getdescr())) + self.call_pure_positions.append(len(self.optimizer._newoperations) - 1) def optimize_GUARD_NO_EXCEPTION(self, op): if self.last_emitted_operation is REMOVED: @@ -111,11 +109,17 @@ class OptPure(Optimization): def get_pure_result(self, key): return self.pure_operations.get(key, None) - def remember_emitting_pure(self, op): - self.emitted_pure_operations[op] = True - def produce_potential_short_preamble_ops(self, sb): - for op in self.emitted_pure_operations: + ops = sb.optimizer._newoperations + for i, op in enumerate(ops): + if op.is_always_pure(): + sb.add_potential(op) + if op.is_ovf() and ops[i + 1].getopnum() == rop.GUARD_NO_OVERFLOW: + sb.add_potential(op) + for i in self.call_pure_positions: + op = ops[i] + assert op.getopnum() == rop.CALL + op = op.copy_and_change(rop.CALL_PURE) sb.add_potential(op) dispatch_opt = make_dispatcher_method(OptPure, 'optimize_', diff --git a/rpython/jit/metainterp/optimizeopt/rewrite.py b/rpython/jit/metainterp/optimizeopt/rewrite.py index f9c4f90783..df85adf13b 100644 --- a/rpython/jit/metainterp/optimizeopt/rewrite.py +++ b/rpython/jit/metainterp/optimizeopt/rewrite.py @@ -117,9 +117,7 @@ class OptRewrite(Optimization): self.make_constant_int(op.result, 0) else: self.emit_operation(op) - # Synthesize the reverse ops for optimize_default to reuse - self.pure(rop.INT_ADD, [op.result, op.getarg(1)], op.getarg(0)) - self.pure(rop.INT_SUB, [op.getarg(0), op.result], op.getarg(1)) + self.optimizer.pure_reverse(op) def optimize_INT_ADD(self, op): v1 = self.getvalue(op.getarg(0)) @@ -132,10 +130,7 @@ class OptRewrite(Optimization): self.make_equal_to(op.result, v1) else: self.emit_operation(op) - self.pure(rop.INT_ADD, [op.getarg(1), op.getarg(0)], op.result) - # Synthesize the reverse op for optimize_default to reuse - self.pure(rop.INT_SUB, [op.result, op.getarg(1)], op.getarg(0)) - self.pure(rop.INT_SUB, [op.result, op.getarg(0)], op.getarg(1)) + self.optimizer.pure_reverse(op) def optimize_INT_MUL(self, op): v1 = self.getvalue(op.getarg(0)) @@ -222,7 +217,7 @@ class OptRewrite(Optimization): )) return self.emit_operation(op) - self.pure(rop.FLOAT_MUL, [arg2, arg1], op.result) + self.optimizer.pure_reverse(op) def optimize_FLOAT_TRUEDIV(self, op): arg1 = op.getarg(0) @@ -244,9 +239,8 @@ class OptRewrite(Optimization): self.emit_operation(op) def optimize_FLOAT_NEG(self, op): - v1 = op.getarg(0) self.emit_operation(op) - self.pure(rop.FLOAT_NEG, [op.result], v1) + self.optimizer.pure_reverse(op) def optimize_guard(self, op, constbox, emit_operation=True): value = self.getvalue(op.getarg(0)) @@ -284,7 +278,7 @@ class OptRewrite(Optimization): raise InvalidLoop('A GUARD_NONNULL (%s) was proven to always fail' % r) self.emit_operation(op) - value.make_nonnull(op) + value.make_nonnull(self.optimizer) def optimize_GUARD_VALUE(self, op): value = self.getvalue(op.getarg(0)) @@ -296,11 +290,12 @@ class OptRewrite(Optimization): else: name = "<unknown>" raise InvalidLoop('A promote of a virtual %s (a recently allocated object) never makes sense!' % name) - if value.get_last_guard(): + old_guard_op = value.get_last_guard(self.optimizer) + if old_guard_op and not isinstance(old_guard_op.getdescr(), + compile.ResumeAtPositionDescr): # there already has been a guard_nonnull or guard_class or # guard_nonnull_class on this value, which is rather silly. # replace the original guard with a guard_value - old_guard_op = value.get_last_guard() if old_guard_op.getopnum() != rop.GUARD_NONNULL: # This is only safe if the class of the guard_value matches the # class of the guard_*_class, otherwise the intermediate ops might @@ -312,19 +307,20 @@ class OptRewrite(Optimization): if not previous_classbox.same_constant(expected_classbox): r = self.optimizer.metainterp_sd.logger_ops.repr_of_resop(op) raise InvalidLoop('A GUARD_VALUE (%s) was proven to always fail' % r) + descr = compile.ResumeGuardValueDescr() op = old_guard_op.copy_and_change(rop.GUARD_VALUE, - args = [old_guard_op.getarg(0), op.getarg(1)]) - self.optimizer.replaces_guard[op] = old_guard_op - # hack hack hack. Change the guard_opnum on - # new_guard_op.getdescr() so that when resuming, - # the operation is not skipped by pyjitpl.py. - descr = op.getdescr() - assert isinstance(descr, compile.ResumeGuardDescr) - descr.guard_opnum = rop.GUARD_VALUE + args = [old_guard_op.getarg(0), op.getarg(1)], + descr = descr) + # Note: we give explicitly a new descr for 'op'; this is why the + # old descr must not be ResumeAtPositionDescr (checked above). + # Better-safe-than-sorry but it should never occur: we should + # not put in short preambles guard_xxx and guard_value + # on the same box. + self.optimizer.replace_guard(op, value) descr.make_a_counter_per_value(op) # to be safe if isinstance(value, PtrOptValue): - value.last_guard = None + value.last_guard_pos = -1 constbox = op.getarg(1) assert isinstance(constbox, Const) self.optimize_guard(op, constbox) @@ -343,7 +339,7 @@ class OptRewrite(Optimization): if realclassbox is not None: assert realclassbox.same_constant(expectedclassbox) return - value.make_constant_class(expectedclassbox, None) + value.make_constant_class(None, expectedclassbox) def optimize_GUARD_CLASS(self, op): value = self.getvalue(op.getarg(0)) @@ -357,24 +353,31 @@ class OptRewrite(Optimization): raise InvalidLoop('A GUARD_CLASS (%s) was proven to always fail' % r) assert isinstance(value, PtrOptValue) - if value.last_guard: + old_guard_op = value.get_last_guard(self.optimizer) + if old_guard_op and not isinstance(old_guard_op.getdescr(), + compile.ResumeAtPositionDescr): # there already has been a guard_nonnull or guard_class or # guard_nonnull_class on this value. - old_guard_op = value.last_guard if old_guard_op.getopnum() == rop.GUARD_NONNULL: # it was a guard_nonnull, which we replace with a # guard_nonnull_class. + descr = compile.ResumeGuardNonnullClassDescr() op = old_guard_op.copy_and_change (rop.GUARD_NONNULL_CLASS, - args = [old_guard_op.getarg(0), op.getarg(1)]) - self.optimizer.replaces_guard[op] = old_guard_op - # hack hack hack. Change the guard_opnum on - # new_guard_op.getdescr() so that when resuming, - # the operation is not skipped by pyjitpl.py. - descr = op.getdescr() - assert isinstance(descr, compile.ResumeGuardDescr) - descr.guard_opnum = rop.GUARD_NONNULL_CLASS + args = [old_guard_op.getarg(0), op.getarg(1)], + descr=descr) + # Note: we give explicitly a new descr for 'op'; this is why the + # old descr must not be ResumeAtPositionDescr (checked above). + # Better-safe-than-sorry but it should never occur: we should + # not put in short preambles guard_nonnull and guard_class + # on the same box. + self.optimizer.replace_guard(op, value) + # not emitting the guard, so we have to pass None to + # make_constant_class, so last_guard_pos is not updated + self.emit_operation(op) + value.make_constant_class(None, expectedclassbox) + return self.emit_operation(op) - value.make_constant_class(expectedclassbox, op) + value.make_constant_class(self.optimizer, expectedclassbox) def optimize_GUARD_NONNULL_CLASS(self, op): value = self.getvalue(op.getarg(0)) @@ -574,11 +577,11 @@ class OptRewrite(Optimization): self.emit_operation(op) def optimize_CAST_PTR_TO_INT(self, op): - self.pure(rop.CAST_INT_TO_PTR, [op.result], op.getarg(0)) + self.optimizer.pure_reverse(op) self.emit_operation(op) def optimize_CAST_INT_TO_PTR(self, op): - self.pure(rop.CAST_PTR_TO_INT, [op.result], op.getarg(0)) + self.optimizer.pure_reverse(op) self.emit_operation(op) def optimize_SAME_AS(self, op): diff --git a/rpython/jit/metainterp/optimizeopt/simplify.py b/rpython/jit/metainterp/optimizeopt/simplify.py index 353a851628..7b7aa13d18 100644 --- a/rpython/jit/metainterp/optimizeopt/simplify.py +++ b/rpython/jit/metainterp/optimizeopt/simplify.py @@ -48,6 +48,7 @@ class OptSimplify(Optimization): def optimize_JUMP(self, op): if not self.unroll: + op = op.clone() descr = op.getdescr() assert isinstance(descr, JitCellToken) if not descr.target_tokens: diff --git a/rpython/jit/metainterp/optimizeopt/test/test_optimizebasic.py b/rpython/jit/metainterp/optimizeopt/test/test_optimizebasic.py index d5249b1371..09301c58da 100644 --- a/rpython/jit/metainterp/optimizeopt/test/test_optimizebasic.py +++ b/rpython/jit/metainterp/optimizeopt/test/test_optimizebasic.py @@ -17,16 +17,16 @@ def test_store_final_boxes_in_guard(): b0 = BoxInt() b1 = BoxInt() opt = optimizeopt.Optimizer(FakeMetaInterpStaticData(LLtypeMixin.cpu), - None) - fdescr = ResumeGuardDescr() - op = ResOperation(rop.GUARD_TRUE, ['dummy'], None, descr=fdescr) + None, None) + op = ResOperation(rop.GUARD_TRUE, ['dummy'], None) # setup rd data fi0 = resume.FrameInfo(None, "code0", 11) - fdescr.rd_frame_info_list = resume.FrameInfo(fi0, "code1", 33) snapshot0 = resume.Snapshot(None, [b0]) - fdescr.rd_snapshot = resume.Snapshot(snapshot0, [b1]) + op.rd_snapshot = resume.Snapshot(snapshot0, [b1]) + op.rd_frame_info_list = resume.FrameInfo(fi0, "code1", 33) # opt.store_final_boxes_in_guard(op, []) + fdescr = op.getdescr() if op.getfailargs() == [b0, b1]: assert list(fdescr.rd_numb.nums) == [tag(1, TAGBOX)] assert list(fdescr.rd_numb.prev.nums) == [tag(0, TAGBOX)] @@ -97,7 +97,8 @@ class BaseTestBasic(BaseTest): enable_opts = "intbounds:rewrite:virtualize:string:earlyforce:pure:heap" def optimize_loop(self, ops, optops, call_pure_results=None): - loop = self.parse(ops) + + loop = self.parse(ops, postprocess=self.postprocess) token = JitCellToken() loop.operations = [ResOperation(rop.LABEL, loop.inputargs, None, descr=TargetToken(token))] + \ loop.operations @@ -108,42 +109,6 @@ class BaseTestBasic(BaseTest): print '\n'.join([str(o) for o in loop.operations]) self.assert_equal(loop, expected) - def setup_method(self, meth=None): - class FailDescr(compile.ResumeGuardDescr): - oparse = None - def _oparser_uses_descr_of_guard(self, oparse, fail_args): - # typically called 3 times: once when parsing 'ops', - # once when parsing 'preamble', once when parsing 'expected'. - self.oparse = oparse - self.rd_frame_info_list, self.rd_snapshot = snapshot(fail_args) - def _clone_if_mutable(self): - assert self is fdescr - return fdescr2 - def __repr__(self): - if self is fdescr: - return 'fdescr' - if self is fdescr2: - return 'fdescr2' - return compile.ResumeGuardDescr.__repr__(self) - # - def snapshot(fail_args, got=[]): - if not got: # only the first time, i.e. when parsing 'ops' - rd_frame_info_list = resume.FrameInfo(None, "code", 11) - rd_snapshot = resume.Snapshot(None, fail_args) - got.append(rd_frame_info_list) - got.append(rd_snapshot) - return got - # - fdescr = instantiate(FailDescr) - self.namespace['fdescr'] = fdescr - fdescr2 = instantiate(FailDescr) - self.namespace['fdescr2'] = fdescr2 - - def teardown_method(self, meth): - self.namespace.pop('fdescr', None) - self.namespace.pop('fdescr2', None) - - class BaseTestOptimizeBasic(BaseTestBasic): @@ -2065,14 +2030,14 @@ class BaseTestOptimizeBasic(BaseTestBasic): def test_merge_guard_nonnull_guard_class(self): ops = """ [p1, i0, i1, i2, p2] - guard_nonnull(p1, descr=fdescr) [i0] + guard_nonnull(p1) [i0] i3 = int_add(i1, i2) guard_class(p1, ConstClass(node_vtable)) [i1] jump(p2, i0, i1, i3, p2) """ expected = """ [p1, i0, i1, i2, p2] - guard_nonnull_class(p1, ConstClass(node_vtable), descr=fdescr) [i0] + guard_nonnull_class(p1, ConstClass(node_vtable)) [i0] i3 = int_add(i1, i2) jump(p2, i0, i1, i3, p2) """ @@ -2082,14 +2047,14 @@ class BaseTestOptimizeBasic(BaseTestBasic): def test_merge_guard_nonnull_guard_value(self): ops = """ [p1, i0, i1, i2, p2] - guard_nonnull(p1, descr=fdescr) [i0] + guard_nonnull(p1) [i0] i3 = int_add(i1, i2) guard_value(p1, ConstPtr(myptr)) [i1] jump(p2, i0, i1, i3, p2) """ expected = """ [p1, i0, i1, i2, p2] - guard_value(p1, ConstPtr(myptr), descr=fdescr) [i0] + guard_value(p1, ConstPtr(myptr)) [i0] i3 = int_add(i1, i2) jump(p2, i0, i1, i3, p2) """ @@ -2099,7 +2064,7 @@ class BaseTestOptimizeBasic(BaseTestBasic): def test_merge_guard_nonnull_guard_class_guard_value(self): ops = """ [p1, i0, i1, i2, p2] - guard_nonnull(p1, descr=fdescr) [i0] + guard_nonnull(p1) [i0] i3 = int_add(i1, i2) guard_class(p1, ConstClass(node_vtable)) [i2] i4 = int_sub(i3, 1) @@ -2108,7 +2073,7 @@ class BaseTestOptimizeBasic(BaseTestBasic): """ expected = """ [p1, i0, i1, i2, p2] - guard_value(p1, ConstPtr(myptr), descr=fdescr) [i0] + guard_value(p1, ConstPtr(myptr)) [i0] i3 = int_add(i1, i2) i4 = int_sub(i3, 1) jump(p2, i0, i1, i4, p2) @@ -2527,7 +2492,7 @@ class BaseTestOptimizeBasic(BaseTestBasic): reader = ResumeDataFakeReader(fdescr, fail_args, MyMetaInterp(self.cpu)) boxes = reader.consume_boxes() - self._verify_fail_args(boxes, fdescr.oparse, expectedtext) + self._verify_fail_args(boxes, self.oparse, expectedtext) def test_expand_fail_1(self): ops = """ @@ -2538,12 +2503,12 @@ class BaseTestOptimizeBasic(BaseTestBasic): i4 = getfield_gc(p1, descr=valuedescr) # i2 = int_add(10, 5) - guard_true(i1, descr=fdescr) [i2, i4] + guard_true(i1) [i2, i4] jump(i1, i4) """ expected = """ [i1, i3] - guard_true(i1, descr=fdescr) [i3] + guard_true(i1) [i3] jump(1, i3) """ self.optimize_loop(ops, expected) @@ -2555,12 +2520,12 @@ class BaseTestOptimizeBasic(BaseTestBasic): p1 = new_with_vtable(ConstClass(node_vtable)) setfield_gc(p1, i2, descr=valuedescr) setfield_gc(p1, p1, descr=nextdescr) - guard_true(i1, descr=fdescr) [p1] + guard_true(i1) [p1] jump(i1, i2) """ expected = """ [i1, i2] - guard_true(i1, descr=fdescr) [i2] + guard_true(i1) [i2] jump(1, i2) """ self.optimize_loop(ops, expected) @@ -2577,12 +2542,12 @@ class BaseTestOptimizeBasic(BaseTestBasic): setfield_gc(p1, p2, descr=nextdescr) setfield_gc(p2, i2, descr=valuedescr) setfield_gc(p2, p3, descr=nextdescr) - guard_true(i1, descr=fdescr) [i3, p1] + guard_true(i1) [i3, p1] jump(i2, i1, i3, p3) """ expected = """ [i1, i2, i3, p3] - guard_true(i1, descr=fdescr) [i3, i2, p3] + guard_true(i1) [i3, i2, p3] jump(i2, 1, i3, p3) """ self.optimize_loop(ops, expected) @@ -2594,7 +2559,6 @@ class BaseTestOptimizeBasic(BaseTestBasic): def test_expand_fail_4(self): for arg in ['p1', 'i2,p1', 'p1,p2', 'p2,p1', 'i2,p1,p2', 'i2,p2,p1']: - self.setup_method() # humpf ops = """ [i1, i2, i3] p1 = new_with_vtable(ConstClass(node_vtable)) @@ -2604,12 +2568,12 @@ class BaseTestOptimizeBasic(BaseTestBasic): setfield_gc(p1, i2, descr=valuedescr) setfield_gc(p1, p2, descr=nextdescr) setfield_gc(p2, i2, descr=valuedescr) - guard_true(i1, descr=fdescr) [i4, i3, %s] + guard_true(i1) [i4, i3, %s] jump(i1, i2, i3) """ expected = """ [i1, i2, i3] - guard_true(i1, descr=fdescr) [i3, i2] + guard_true(i1) [i3, i2] jump(1, i2, i3) """ self.optimize_loop(ops % arg, expected) @@ -2627,12 +2591,12 @@ class BaseTestOptimizeBasic(BaseTestBasic): setfield_gc(p1, p2, descr=nextdescr) setfield_gc(p2, i2, descr=valuedescr) setfield_gc(p2, p1, descr=nextdescr) # a cycle - guard_true(i1, descr=fdescr) [i3, i4, p1, p2] + guard_true(i1) [i3, i4, p1, p2] jump(i2, i1, i3, i4) """ expected = """ [i1, i2, i3, i4] - guard_true(i1, descr=fdescr) [i3, i4, i2] + guard_true(i1) [i3, i4, i2] jump(i2, 1, i3, i4) """ self.optimize_loop(ops, expected) @@ -2644,14 +2608,14 @@ class BaseTestOptimizeBasic(BaseTestBasic): def test_expand_fail_6(self): ops = """ [p0, i0, i1] - guard_true(i0, descr=fdescr) [p0] + guard_true(i0) [p0] p1 = new_with_vtable(ConstClass(node_vtable)) setfield_gc(p1, i1, descr=valuedescr) jump(p1, i1, i1) """ expected = """ [i1b, i0, i1] - guard_true(i0, descr=fdescr) [i1b] + guard_true(i0) [i1b] jump(i1, i1, i1) """ py.test.skip("XXX") @@ -2667,13 +2631,13 @@ class BaseTestOptimizeBasic(BaseTestBasic): p1 = new_array(3, descr=arraydescr) setarrayitem_gc(p1, 1, i1, descr=arraydescr) setarrayitem_gc(p1, 0, 25, descr=arraydescr) - guard_true(i1, descr=fdescr) [p1] + guard_true(i1) [p1] i2 = getarrayitem_gc(p1, 1, descr=arraydescr) jump(i2) """ expected = """ [i1] - guard_true(i1, descr=fdescr) [i1] + guard_true(i1) [i1] jump(1) """ self.optimize_loop(ops, expected) @@ -2687,14 +2651,14 @@ class BaseTestOptimizeBasic(BaseTestBasic): p2 = new(descr=ssize) setfield_gc(p2, i1, descr=adescr) setfield_gc(p2, p1, descr=bdescr) - guard_true(i1, descr=fdescr) [p2] + guard_true(i1) [p2] i3 = getfield_gc(p2, descr=adescr) p3 = getfield_gc(p2, descr=bdescr) jump(i3, p3) """ expected = """ [i1, p1] - guard_true(i1, descr=fdescr) [i1, p1] + guard_true(i1) [i1, p1] jump(1, p1) """ self.optimize_loop(ops, expected) @@ -2711,7 +2675,7 @@ class BaseTestOptimizeBasic(BaseTestBasic): setfield_gc(p5s, i2, descr=adescr) setfield_gc(p5s, p7v, descr=bdescr) setarrayitem_gc(p1a, 1, p5s, descr=arraydescr2) - guard_true(i1, descr=fdescr) [p1a] + guard_true(i1) [p1a] p2s = new(descr=ssize) p3v = new_with_vtable(ConstClass(node_vtable)) p4a = new_array(2, descr=arraydescr2) @@ -2723,7 +2687,7 @@ class BaseTestOptimizeBasic(BaseTestBasic): """ expected = """ [i1, ia, iv, pnull, i2] - guard_true(i1, descr=fdescr) [ia, iv, i2] + guard_true(i1) [ia, iv, i2] jump(1, 1, i2, NULL, i2) """ py.test.skip("XXX") @@ -2749,14 +2713,14 @@ class BaseTestOptimizeBasic(BaseTestBasic): p2 = new_with_vtable(ConstClass(node_vtable)) setfield_gc(p2, i2, descr=valuedescr) setfield_gc(p1, p2, descr=nextdescr) - guard_true(i3, descr=fdescr) [] + guard_true(i3) [] i4 = int_neg(i2) setfield_gc(p1, NULL, descr=nextdescr) jump(p1, i2, i4) """ expected = """ [p1, i2, i3] - guard_true(i3, descr=fdescr) [p1, i2] + guard_true(i3) [p1, i2] i4 = int_neg(i2) setfield_gc(p1, NULL, descr=nextdescr) jump(p1, i2, i4) @@ -2774,14 +2738,14 @@ class BaseTestOptimizeBasic(BaseTestBasic): p2 = new_with_vtable(ConstClass(node_vtable)) setfield_gc(p2, i2, descr=valuedescr) setfield_gc(ConstPtr(myptr), p2, descr=nextdescr) - guard_true(i3, descr=fdescr) [] + guard_true(i3) [] i4 = int_neg(i2) setfield_gc(ConstPtr(myptr), NULL, descr=nextdescr) jump(i2, i4) """ expected = """ [i2, i3] - guard_true(i3, descr=fdescr) [i2] + guard_true(i3) [i2] i4 = int_neg(i2) setfield_gc(ConstPtr(myptr), NULL, descr=nextdescr) jump(i2, i4) @@ -3096,7 +3060,7 @@ class BaseTestOptimizeBasic(BaseTestBasic): p2 = virtual_ref(p1, 2) setfield_gc(p0, p2, descr=nextdescr) call_may_force(i1, descr=mayforcevirtdescr) - guard_not_forced(descr=fdescr) [p2, p1] + guard_not_forced() [p2, p1] virtual_ref_finish(p2, p1) setfield_gc(p0, NULL, descr=nextdescr) jump(p0, i1) @@ -3111,7 +3075,7 @@ class BaseTestOptimizeBasic(BaseTestBasic): setfield_gc(p0, p2, descr=nextdescr) # call_may_force(i1, descr=mayforcevirtdescr) - guard_not_forced(descr=fdescr) [p2, i1] + guard_not_forced() [p2, i1] # setfield_gc(p0, NULL, descr=nextdescr) p1 = new_with_vtable(ConstClass(node_vtable)) @@ -3142,7 +3106,7 @@ class BaseTestOptimizeBasic(BaseTestBasic): p2 = virtual_ref(p1, 2) setfield_gc(p0, p2, descr=refdescr) call(i1, descr=nonwritedescr) - guard_no_exception(descr=fdescr) [p2, p1] + guard_no_exception() [p2, p1] virtual_ref_finish(p2, p1) setfield_gc(p0, NULL, descr=refdescr) jump(p0, i1) @@ -3151,7 +3115,7 @@ class BaseTestOptimizeBasic(BaseTestBasic): [p0, i1] p3 = force_token() call(i1, descr=nonwritedescr) - guard_no_exception(descr=fdescr) [p3, i1, p0] + guard_no_exception() [p3, i1, p0] setfield_gc(p0, NULL, descr=refdescr) jump(p0, i1) """ @@ -5563,6 +5527,5 @@ class BaseTestOptimizeBasic(BaseTestBasic): """ self.optimize_loop(ops, ops) - class TestLLtype(BaseTestOptimizeBasic, LLtypeMixin): pass diff --git a/rpython/jit/metainterp/optimizeopt/test/test_optimizeopt.py b/rpython/jit/metainterp/optimizeopt/test/test_optimizeopt.py index 63e9e09ac5..458a35db7c 100644 --- a/rpython/jit/metainterp/optimizeopt/test/test_optimizeopt.py +++ b/rpython/jit/metainterp/optimizeopt/test/test_optimizeopt.py @@ -45,7 +45,7 @@ class BaseTestWithUnroll(BaseTest): def optimize_loop(self, ops, expected, expected_preamble=None, call_pure_results=None, expected_short=None): - loop = self.parse(ops) + loop = self.parse(ops, postprocess=self.postprocess) if expected != "crash!": expected = self.parse(expected) if expected_preamble: @@ -96,41 +96,6 @@ class BaseTestWithUnroll(BaseTest): class OptimizeOptTest(BaseTestWithUnroll): - def setup_method(self, meth=None): - class FailDescr(compile.ResumeGuardDescr): - oparse = None - def _oparser_uses_descr_of_guard(self, oparse, fail_args): - # typically called 3 times: once when parsing 'ops', - # once when parsing 'preamble', once when parsing 'expected'. - self.oparse = oparse - self.rd_frame_info_list, self.rd_snapshot = snapshot(fail_args) - def _clone_if_mutable(self): - assert self is fdescr - return fdescr2 - def __repr__(self): - if self is fdescr: - return 'fdescr' - if self is fdescr2: - return 'fdescr2' - return compile.ResumeGuardDescr.__repr__(self) - # - def snapshot(fail_args, got=[]): - if not got: # only the first time, i.e. when parsing 'ops' - rd_frame_info_list = resume.FrameInfo(None, "code", 11) - rd_snapshot = resume.Snapshot(None, fail_args) - got.append(rd_frame_info_list) - got.append(rd_snapshot) - return got - # - fdescr = instantiate(FailDescr) - self.namespace['fdescr'] = fdescr - fdescr2 = instantiate(FailDescr) - self.namespace['fdescr2'] = fdescr2 - - def teardown_method(self, meth): - self.namespace.pop('fdescr', None) - self.namespace.pop('fdescr2', None) - def test_simple(self): ops = """ [] @@ -3004,20 +2969,20 @@ class OptimizeOptTest(BaseTestWithUnroll): def test_merge_guard_nonnull_guard_class(self): ops = """ [p1, i0, i1, i2, p2] - guard_nonnull(p1, descr=fdescr) [i0] + guard_nonnull(p1) [i0] i3 = int_add(i1, i2) guard_class(p1, ConstClass(node_vtable)) [i1] jump(p2, i0, i1, i3, p2) """ preamble = """ [p1, i0, i1, i2, p2] - guard_nonnull_class(p1, ConstClass(node_vtable), descr=fdescr) [i0] + guard_nonnull_class(p1, ConstClass(node_vtable)) [i0] i3 = int_add(i1, i2) jump(p2, i0, i1, i3) """ expected = """ [p2, i0, i1, i2] - guard_nonnull_class(p2, ConstClass(node_vtable), descr=fdescr2) [i0] + guard_nonnull_class(p2, ConstClass(node_vtable)) [i0] i3 = int_add(i1, i2) jump(p2, i0, i1, i3) """ @@ -3027,20 +2992,20 @@ class OptimizeOptTest(BaseTestWithUnroll): def test_merge_guard_nonnull_guard_value(self): ops = """ [p1, i0, i1, i2, p2] - guard_nonnull(p1, descr=fdescr) [i0] + guard_nonnull(p1) [i0] i3 = int_add(i1, i2) guard_value(p1, ConstPtr(myptr)) [i1] jump(p2, i0, i1, i3, p2) """ preamble = """ [p1, i0, i1, i2, p2] - guard_value(p1, ConstPtr(myptr), descr=fdescr) [i0] + guard_value(p1, ConstPtr(myptr)) [i0] i3 = int_add(i1, i2) jump(p2, i0, i1, i3) """ expected = """ [p2, i0, i1, i2] - guard_value(p2, ConstPtr(myptr), descr=fdescr2) [i0] + guard_value(p2, ConstPtr(myptr)) [i0] i3 = int_add(i1, i2) jump(ConstPtr(myptr), i0, i1, i3) """ @@ -3050,7 +3015,7 @@ class OptimizeOptTest(BaseTestWithUnroll): def test_merge_guard_nonnull_guard_class_guard_value(self): ops = """ [p1, i0, i1, i2, p2] - guard_nonnull(p1, descr=fdescr) [i0] + guard_nonnull(p1) [i0] i3 = int_add(i1, i2) guard_class(p1, ConstClass(node_vtable)) [i2] i4 = int_sub(i3, 1) @@ -3059,14 +3024,14 @@ class OptimizeOptTest(BaseTestWithUnroll): """ preamble = """ [p1, i0, i1, i2, p2] - guard_value(p1, ConstPtr(myptr), descr=fdescr) [i0] + guard_value(p1, ConstPtr(myptr)) [i0] i3 = int_add(i1, i2) i4 = int_sub(i3, 1) jump(p2, i0, i1, i4) """ expected = """ [p2, i0, i1, i2] - guard_value(p2, ConstPtr(myptr), descr=fdescr2) [i0] + guard_value(p2, ConstPtr(myptr)) [i0] i3 = int_add(i1, i2) i4 = int_sub(i3, 1) jump(ConstPtr(myptr), i0, i1, i4) @@ -3860,7 +3825,7 @@ class OptimizeOptTest(BaseTestWithUnroll): p2 = virtual_ref(p1, 2) setfield_gc(p0, p2, descr=nextdescr) call_may_force(i1, descr=mayforcevirtdescr) - guard_not_forced(descr=fdescr) [p2, p1] + guard_not_forced() [p2, p1] virtual_ref_finish(p2, p1) setfield_gc(p0, NULL, descr=nextdescr) jump(p0, i1) @@ -3875,7 +3840,7 @@ class OptimizeOptTest(BaseTestWithUnroll): setfield_gc(p0, p2, descr=nextdescr) # call_may_force(i1, descr=mayforcevirtdescr) - guard_not_forced(descr=fdescr2) [p2, i1] + guard_not_forced() [p2, i1] # setfield_gc(p0, NULL, descr=nextdescr) p1 = new_with_vtable(ConstClass(node_vtable)) @@ -3906,7 +3871,7 @@ class OptimizeOptTest(BaseTestWithUnroll): p2 = virtual_ref(p1, 2) setfield_gc(p0, p2, descr=refdescr) call(i1, descr=nonwritedescr) - guard_no_exception(descr=fdescr) [p2, p1] + guard_no_exception() [p2, p1] virtual_ref_finish(p2, p1) setfield_gc(p0, NULL, descr=refdescr) escape() @@ -3916,7 +3881,7 @@ class OptimizeOptTest(BaseTestWithUnroll): [p0, i1] p3 = force_token() call(i1, descr=nonwritedescr) - guard_no_exception(descr=fdescr) [p3, i1, p0] + guard_no_exception() [p3, i1, p0] setfield_gc(p0, NULL, descr=refdescr) escape() jump(p0, i1) @@ -3925,7 +3890,7 @@ class OptimizeOptTest(BaseTestWithUnroll): [p0, i1] p3 = force_token() call(i1, descr=nonwritedescr) - guard_no_exception(descr=fdescr2) [p3, i1, p0] + guard_no_exception() [p3, i1, p0] setfield_gc(p0, NULL, descr=refdescr) escape() jump(p0, i1) @@ -7290,11 +7255,9 @@ class OptimizeOptTest(BaseTestWithUnroll): short = """ [p0] p1 = getfield_gc(p0, descr=nextdescr) - guard_nonnull(p1) [] - guard_class(p1, ConstClass(node_vtable)) [] + guard_nonnull_class(p1, ConstClass(node_vtable)) [] p2 = getfield_gc(p1, descr=nextdescr) - guard_nonnull(p2) [] - guard_class(p2, ConstClass(node_vtable)) [] + guard_nonnull_class(p2, ConstClass(node_vtable)) [] jump(p0) """ self.optimize_loop(ops, expected, expected_short=short) diff --git a/rpython/jit/metainterp/optimizeopt/test/test_util.py b/rpython/jit/metainterp/optimizeopt/test/test_util.py index 71e8129289..0f158e1852 100644 --- a/rpython/jit/metainterp/optimizeopt/test/test_util.py +++ b/rpython/jit/metainterp/optimizeopt/test/test_util.py @@ -8,12 +8,13 @@ from rpython.rtyper.rclass import ( from rpython.jit.backend.llgraph import runner from rpython.jit.metainterp.history import (BoxInt, BoxPtr, ConstInt, ConstPtr, Const, TreeLoop, AbstractDescr, - JitCellToken, TargetToken) + JitCellToken, TargetToken, + BasicFinalDescr) from rpython.jit.metainterp.optimizeopt.util import sort_descrs, equaloplists from rpython.jit.metainterp.optimize import InvalidLoop from rpython.jit.codewriter.effectinfo import EffectInfo from rpython.jit.codewriter.heaptracker import register_known_gctype, adr2int -from rpython.jit.tool.oparser import parse, pure_parse +from rpython.jit.tool.oparser import OpParser, pure_parse from rpython.jit.metainterp.quasiimmut import QuasiImmutDescr from rpython.jit.metainterp import compile, resume, history from rpython.jit.metainterp.jitprof import EmptyProfiler @@ -348,42 +349,35 @@ class Storage(compile.ResumeGuardDescr): def store_final_boxes(self, op, boxes, metainterp_sd): op.setfailargs(boxes) def __eq__(self, other): - return type(self) is type(other) # xxx obscure - def clone_if_mutable(self): - res = Storage(self.metainterp_sd, self.original_greenkey) - self.copy_all_attributes_into(res) - return res + return True # screw this + #return type(self) is type(other) # xxx obscure def _sortboxes(boxes): _kind2count = {history.INT: 1, history.REF: 2, history.FLOAT: 3} return sorted(boxes, key=lambda box: _kind2count[box.type]) +final_descr = BasicFinalDescr() + class BaseTest(object): - def parse(self, s, boxkinds=None, want_fail_descr=True): - if want_fail_descr: - invent_fail_descr = self.invent_fail_descr - else: - invent_fail_descr = lambda *args: None - return parse(s, self.cpu, self.namespace, - type_system=self.type_system, - boxkinds=boxkinds, - invent_fail_descr=invent_fail_descr) + def parse(self, s, boxkinds=None, want_fail_descr=True, postprocess=None): + self.oparse = OpParser(s, self.cpu, self.namespace, 'lltype', + boxkinds, + None, False, postprocess) + return self.oparse.parse() + + def postprocess(self, op): + if op.is_guard(): + op.rd_snapshot = resume.Snapshot(None, op.getfailargs()) + op.rd_frame_info_list = resume.FrameInfo(None, "code", 11) def add_guard_future_condition(self, res): # invent a GUARD_FUTURE_CONDITION to not have to change all tests if res.operations[-1].getopnum() == rop.JUMP: - guard = ResOperation(rop.GUARD_FUTURE_CONDITION, [], None, descr=self.invent_fail_descr(None, -1, [])) + guard = ResOperation(rop.GUARD_FUTURE_CONDITION, [], None) + guard.rd_snapshot = resume.Snapshot(None, []) res.operations.insert(-1, guard) - def invent_fail_descr(self, model, opnum, fail_args): - if fail_args is None: - return None - descr = Storage() - descr.rd_frame_info_list = resume.FrameInfo(None, "code", 11) - descr.rd_snapshot = resume.Snapshot(None, _sortboxes(fail_args)) - return descr - def assert_equal(self, optimized, expected, text_right=None): from rpython.jit.metainterp.optimizeopt.util import equaloplists assert len(optimized.inputargs) == len(expected.inputargs) @@ -410,7 +404,8 @@ class BaseTest(object): if hasattr(self, 'callinfocollection'): metainterp_sd.callinfocollection = self.callinfocollection # - return optimize_trace(metainterp_sd, loop, self.enable_opts, + return optimize_trace(metainterp_sd, None, loop, + self.enable_opts, start_state=start_state, export_state=export_state) diff --git a/rpython/jit/metainterp/optimizeopt/test/test_virtualstate.py b/rpython/jit/metainterp/optimizeopt/test/test_virtualstate.py index 845f006784..480de3d21b 100644 --- a/rpython/jit/metainterp/optimizeopt/test/test_virtualstate.py +++ b/rpython/jit/metainterp/optimizeopt/test/test_virtualstate.py @@ -15,6 +15,7 @@ from rpython.jit.metainterp.optimizeopt.virtualize import (VirtualValue, from rpython.jit.metainterp.history import TreeLoop, JitCellToken from rpython.jit.metainterp.optimizeopt.test.test_optimizeopt import FakeMetaInterpStaticData from rpython.jit.metainterp.resoperation import ResOperation, rop +from rpython.jit.metainterp import resume class BaseTestGenerateGuards(BaseTest): @@ -124,12 +125,12 @@ class BaseTestGenerateGuards(BaseTest): ptr = PtrOptValue(BoxPtr()) nonnull = PtrOptValue(BoxPtr()) - nonnull.make_nonnull(0) + nonnull.make_nonnull(None) knownclass = PtrOptValue(BoxPtr()) clsbox = self.cpu.ts.cls_of_box(BoxPtr(self.myptr)) - knownclass.make_constant_class(clsbox, 0) + knownclass.make_constant_class(None, clsbox) const = PtrOptValue(BoxPtr) - const.make_constant_class(clsbox, 0) + const.make_constant_class(None, clsbox) const.make_constant(ConstPtr(self.myptr)) inorder = [ptr, nonnull, knownclass, const] for i in range(len(inorder)): @@ -180,18 +181,18 @@ class BaseTestGenerateGuards(BaseTest): def test_known_class_generalization(self): knownclass1 = PtrOptValue(BoxPtr()) - knownclass1.make_constant_class(ConstPtr(self.myptr), 0) + knownclass1.make_constant_class(None, ConstPtr(self.myptr)) info1 = NotVirtualStateInfo(knownclass1) info1.position = 0 knownclass2 = PtrOptValue(BoxPtr()) - knownclass2.make_constant_class(ConstPtr(self.myptr), 0) + knownclass2.make_constant_class(None, ConstPtr(self.myptr)) info2 = NotVirtualStateInfo(knownclass2) info2.position = 0 self.check_no_guards(info1, info2) self.check_no_guards(info2, info1) knownclass3 = PtrOptValue(BoxPtr()) - knownclass3.make_constant_class(ConstPtr(self.myptr2), 0) + knownclass3.make_constant_class(None, ConstPtr(self.myptr2)) info3 = NotVirtualStateInfo(knownclass3) info3.position = 0 self.check_invalid(info1, info3) @@ -220,11 +221,11 @@ class BaseTestGenerateGuards(BaseTest): knownclass_val = PtrOptValue(self.nodebox) classbox = self.cpu.ts.cls_of_box(self.nodebox) - knownclass_val.make_constant_class(classbox, -1) + knownclass_val.make_constant_class(None, classbox,) knownclass_info = NotVirtualStateInfo(knownclass_val) knownclass2_val = PtrOptValue(self.nodebox2) classbox = self.cpu.ts.cls_of_box(self.nodebox2) - knownclass2_val.make_constant_class(classbox, -1) + knownclass2_val.make_constant_class(None, classbox) knownclass2_info = NotVirtualStateInfo(knownclass2_val) constant_val = IntOptValue(BoxInt()) @@ -284,8 +285,7 @@ class BaseTestGenerateGuards(BaseTest): # knownclass unknown expected = """ [p0] - guard_nonnull(p0) [] - guard_class(p0, ConstClass(node_vtable)) [] + guard_nonnull_class(p0, ConstClass(node_vtable)) [] """ self.guards(knownclass_info, unknown_info, unknown_val, expected) self.check_invalid(knownclass_info, unknown_info, unknownnull_val) @@ -384,13 +384,12 @@ class BaseTestGenerateGuards(BaseTest): def test_known_class(self): value1 = PtrOptValue(self.nodebox) classbox = self.cpu.ts.cls_of_box(self.nodebox) - value1.make_constant_class(classbox, -1) + value1.make_constant_class(None, classbox) info1 = NotVirtualStateInfo(value1) info2 = NotVirtualStateInfo(PtrOptValue(self.nodebox)) expected = """ [p0] - guard_nonnull(p0) [] - guard_class(p0, ConstClass(node_vtable)) [] + guard_nonnull_class(p0, ConstClass(node_vtable)) [] """ self.guards(info1, info2, self.nodebox, expected) self.check_invalid(info1, info2, BoxPtr()) @@ -398,13 +397,12 @@ class BaseTestGenerateGuards(BaseTest): def test_known_class_value(self): value1 = PtrOptValue(self.nodebox) classbox = self.cpu.ts.cls_of_box(self.nodebox) - value1.make_constant_class(classbox, -1) + value1.make_constant_class(None, classbox) box = self.nodebox guards = value1.make_guards(box) expected = """ [p0] - guard_nonnull(p0) [] - guard_class(p0, ConstClass(node_vtable)) [] + guard_nonnull_class(p0, ConstClass(node_vtable)) [] """ self.compare(guards, expected, [box]) @@ -422,7 +420,7 @@ class BaseTestGenerateGuards(BaseTest): def test_equal_inputargs(self): value = PtrOptValue(self.nodebox) classbox = self.cpu.ts.cls_of_box(self.nodebox) - value.make_constant_class(classbox, -1) + value.make_constant_class(None, classbox) knownclass_info = NotVirtualStateInfo(value) vstate1 = VirtualState([knownclass_info, knownclass_info]) assert vstate1.generalization_of(vstate1) @@ -443,8 +441,7 @@ class BaseTestGenerateGuards(BaseTest): expected = """ [p0] - guard_nonnull(p0) [] - guard_class(p0, ConstClass(node_vtable)) [] + guard_nonnull_class(p0, ConstClass(node_vtable)) [] """ state = vstate1.generate_guards(vstate2, [value, value], self.cpu) self.compare(state.extra_guards, expected, [self.nodebox]) @@ -460,7 +457,7 @@ class BaseTestGenerateGuards(BaseTest): def test_generate_guards_on_virtual_fields_matches_array(self): innervalue1 = PtrOptValue(self.nodebox) constclassbox = self.cpu.ts.cls_of_box(self.nodebox) - innervalue1.make_constant_class(constclassbox, -1) + innervalue1.make_constant_class(None, constclassbox) innerinfo1 = NotVirtualStateInfo(innervalue1) innerinfo1.position = 1 innerinfo2 = NotVirtualStateInfo(PtrOptValue(self.nodebox)) @@ -479,15 +476,14 @@ class BaseTestGenerateGuards(BaseTest): expected = """ [p0] - guard_nonnull(p0) [] - guard_class(p0, ConstClass(node_vtable)) [] + guard_nonnull_class(p0, ConstClass(node_vtable)) [] """ self.guards(info1, info2, value1, expected, [self.nodebox]) def test_generate_guards_on_virtual_fields_matches_instance(self): innervalue1 = PtrOptValue(self.nodebox) constclassbox = self.cpu.ts.cls_of_box(self.nodebox) - innervalue1.make_constant_class(constclassbox, -1) + innervalue1.make_constant_class(None, constclassbox) innerinfo1 = NotVirtualStateInfo(innervalue1) innerinfo1.position = 1 innerinfo2 = NotVirtualStateInfo(PtrOptValue(self.nodebox)) @@ -504,15 +500,14 @@ class BaseTestGenerateGuards(BaseTest): expected = """ [p0] - guard_nonnull(p0) [] - guard_class(p0, ConstClass(node_vtable)) [] + guard_nonnull_class(p0, ConstClass(node_vtable)) [] """ self.guards(info1, info2, value1, expected, [self.nodebox]) def test_generate_guards_on_virtual_fields_matches_struct(self): innervalue1 = PtrOptValue(self.nodebox) constclassbox = self.cpu.ts.cls_of_box(self.nodebox) - innervalue1.make_constant_class(constclassbox, -1) + innervalue1.make_constant_class(None, constclassbox) innerinfo1 = NotVirtualStateInfo(innervalue1) innerinfo1.position = 1 innerinfo2 = NotVirtualStateInfo(PtrOptValue(self.nodebox)) @@ -531,15 +526,14 @@ class BaseTestGenerateGuards(BaseTest): expected = """ [p0] - guard_nonnull(p0) [] - guard_class(p0, ConstClass(node_vtable)) [] + guard_nonnull_class(p0, ConstClass(node_vtable)) [] """ self.guards(info1, info2, value1, expected, [self.nodebox]) def test_generate_guards_on_virtual_fields_matches_arraystruct(self): innervalue1 = PtrOptValue(self.nodebox) constclassbox = self.cpu.ts.cls_of_box(self.nodebox) - innervalue1.make_constant_class(constclassbox, -1) + innervalue1.make_constant_class(None, constclassbox) innerinfo1 = NotVirtualStateInfo(innervalue1) innerinfo1.position = 1 innerinfo2 = NotVirtualStateInfo(PtrOptValue(self.nodebox)) @@ -559,8 +553,7 @@ class BaseTestGenerateGuards(BaseTest): expected = """ [p0] - guard_nonnull(p0) [] - guard_class(p0, ConstClass(node_vtable)) [] + guard_nonnull_class(p0, ConstClass(node_vtable)) [] """ self.guards(info1, info2, value1, expected, [self.nodebox]) @@ -571,7 +564,7 @@ class BaseTestGenerateGuards(BaseTest): info1 = VirtualStateInfo(ConstInt(42), [1, 2]) value = PtrOptValue(self.nodebox) classbox = self.cpu.ts.cls_of_box(self.nodebox) - value.make_constant_class(classbox, -1) + value.make_constant_class(None, classbox) knownclass_info = NotVirtualStateInfo(value) info1.fieldstate = [knownclass_info, knownclass_info] vstate1 = VirtualState([info1]) @@ -599,7 +592,7 @@ class BaseTestGenerateGuards(BaseTest): info1 = VirtualStateInfo(ConstInt(42), [1, 2]) value = PtrOptValue(self.nodebox) classbox = self.cpu.ts.cls_of_box(self.nodebox) - value.make_constant_class(classbox, -1) + value.make_constant_class(None, classbox) knownclass_info = NotVirtualStateInfo(value) info1.fieldstate = [knownclass_info, knownclass_info] vstate1 = VirtualState([info1]) @@ -608,7 +601,7 @@ class BaseTestGenerateGuards(BaseTest): info2 = VirtualStateInfo(ConstInt(42), [1, 2]) value = PtrOptValue(self.nodebox2) classbox = self.cpu.ts.cls_of_box(self.nodebox2) - value.make_constant_class(classbox, -1) + value.make_constant_class(None, classbox) knownclass_info = NotVirtualStateInfo(value) info2.fieldstate = [knownclass_info, knownclass_info] vstate2 = VirtualState([info2]) @@ -621,7 +614,7 @@ class BaseTestGenerateGuards(BaseTest): info1 = VirtualStateInfo(ConstInt(42), [10, 20]) value = PtrOptValue(self.nodebox) classbox = self.cpu.ts.cls_of_box(self.nodebox) - value.make_constant_class(classbox, -1) + value.make_constant_class(None, classbox) knownclass_info = NotVirtualStateInfo(value) info1.fieldstate = [knownclass_info, knownclass_info] vstate1 = VirtualState([info1]) @@ -630,7 +623,7 @@ class BaseTestGenerateGuards(BaseTest): info2 = VirtualStateInfo(ConstInt(42), [1, 2]) value = PtrOptValue(self.nodebox2) classbox = self.cpu.ts.cls_of_box(self.nodebox2) - value.make_constant_class(classbox, -1) + value.make_constant_class(None, classbox) knownclass_info = NotVirtualStateInfo(value) info2.fieldstate = [knownclass_info, knownclass_info] vstate2 = VirtualState([info2]) @@ -643,7 +636,7 @@ class BaseTestGenerateGuards(BaseTest): info1 = VirtualStateInfo(ConstInt(42), [1, 2]) value = PtrOptValue(self.nodebox) classbox = self.cpu.ts.cls_of_box(self.nodebox) - value.make_constant_class(classbox, -1) + value.make_constant_class(None, classbox) knownclass_info = NotVirtualStateInfo(value) info1.fieldstate = [knownclass_info, knownclass_info] vstate1 = VirtualState([info1]) @@ -652,7 +645,7 @@ class BaseTestGenerateGuards(BaseTest): info2 = VirtualStateInfo(ConstInt(7), [1, 2]) value = PtrOptValue(self.nodebox2) classbox = self.cpu.ts.cls_of_box(self.nodebox2) - value.make_constant_class(classbox, -1) + value.make_constant_class(None, classbox) knownclass_info = NotVirtualStateInfo(value) info2.fieldstate = [knownclass_info, knownclass_info] vstate2 = VirtualState([info2]) @@ -665,7 +658,7 @@ class BaseTestGenerateGuards(BaseTest): info1 = VirtualStateInfo(ConstInt(42), [1, 2]) value = PtrOptValue(self.nodebox) classbox = self.cpu.ts.cls_of_box(self.nodebox) - value.make_constant_class(classbox, -1) + value.make_constant_class(None, classbox) knownclass_info = NotVirtualStateInfo(value) info1.fieldstate = [knownclass_info, knownclass_info] vstate1 = VirtualState([info1]) @@ -682,7 +675,7 @@ class BaseTestGenerateGuards(BaseTest): info1 = VArrayStateInfo(42) value = PtrOptValue(self.nodebox) classbox = self.cpu.ts.cls_of_box(self.nodebox) - value.make_constant_class(classbox, -1) + value.make_constant_class(None, classbox) knownclass_info = NotVirtualStateInfo(value) info1.fieldstate = [knownclass_info, knownclass_info] vstate1 = VirtualState([info1]) @@ -691,7 +684,7 @@ class BaseTestGenerateGuards(BaseTest): info2 = VArrayStateInfo(42) value = PtrOptValue(self.nodebox2) classbox = self.cpu.ts.cls_of_box(self.nodebox2) - value.make_constant_class(classbox, -1) + value.make_constant_class(None, classbox) knownclass_info = NotVirtualStateInfo(value) info2.fieldstate = [knownclass_info, knownclass_info] vstate2 = VirtualState([info2]) @@ -704,7 +697,7 @@ class BaseTestGenerateGuards(BaseTest): info1 = VArrayStateInfo(42) value = PtrOptValue(self.nodebox) classbox = self.cpu.ts.cls_of_box(self.nodebox) - value.make_constant_class(classbox, -1) + value.make_constant_class(None, classbox) knownclass_info = NotVirtualStateInfo(value) info1.fieldstate = [knownclass_info, knownclass_info] vstate1 = VirtualState([info1]) @@ -713,7 +706,7 @@ class BaseTestGenerateGuards(BaseTest): info2 = VArrayStateInfo(42) value = PtrOptValue(self.nodebox) classbox = self.cpu.ts.cls_of_box(self.nodebox) - value.make_constant_class(classbox, -1) + value.make_constant_class(None, classbox) knownclass_info = NotVirtualStateInfo(value) info2.fieldstate = [knownclass_info] vstate2 = VirtualState([info2]) @@ -726,7 +719,7 @@ class BaseTestGenerateGuards(BaseTest): info1 = VArrayStateInfo(42) value = PtrOptValue(self.nodebox) classbox = self.cpu.ts.cls_of_box(self.nodebox) - value.make_constant_class(classbox, -1) + value.make_constant_class(None, classbox) knownclass_info = NotVirtualStateInfo(value) info1.fieldstate = [knownclass_info, knownclass_info] vstate1 = VirtualState([info1]) @@ -735,7 +728,7 @@ class BaseTestGenerateGuards(BaseTest): info2 = VArrayStateInfo(7) value = PtrOptValue(self.nodebox2) classbox = self.cpu.ts.cls_of_box(self.nodebox2) - value.make_constant_class(classbox, -1) + value.make_constant_class(None, classbox) knownclass_info = NotVirtualStateInfo(value) info2.fieldstate = [knownclass_info, knownclass_info] vstate2 = VirtualState([info2]) @@ -748,7 +741,7 @@ class BaseTestGenerateGuards(BaseTest): info1 = VArrayStateInfo(42) value = PtrOptValue(self.nodebox) classbox = self.cpu.ts.cls_of_box(self.nodebox) - value.make_constant_class(classbox, -1) + value.make_constant_class(None, classbox) knownclass_info = NotVirtualStateInfo(value) info1.fieldstate = [knownclass_info, knownclass_info] vstate1 = VirtualState([info1]) @@ -765,7 +758,7 @@ class BaseTestGenerateGuards(BaseTest): def test_crash_varay_clear(self): innervalue1 = PtrOptValue(self.nodebox) constclassbox = self.cpu.ts.cls_of_box(self.nodebox) - innervalue1.make_constant_class(constclassbox, -1) + innervalue1.make_constant_class(None, constclassbox) innerinfo1 = NotVirtualStateInfo(innervalue1) innerinfo1.position = 1 innerinfo1.position_in_notvirtuals = 0 @@ -798,14 +791,15 @@ class BaseTestBridges(BaseTest): if hasattr(self, 'callinfocollection'): metainterp_sd.callinfocollection = self.callinfocollection # - optimize_trace(metainterp_sd, bridge, self.enable_opts) + optimize_trace(metainterp_sd, None, bridge, self.enable_opts) def optimize_bridge(self, loops, bridge, expected, expected_target='Loop', **boxvalues): if isinstance(loops, str): loops = (loops, ) - loops = [self.parse(loop) for loop in loops] - bridge = self.parse(bridge) + loops = [self.parse(loop, postprocess=self.postprocess) + for loop in loops] + bridge = self.parse(bridge, postprocess=self.postprocess) self.add_guard_future_condition(bridge) for loop in loops: loop.preamble = self.unroll_and_optimize(loop) diff --git a/rpython/jit/metainterp/optimizeopt/test/test_zdisable_opts.py b/rpython/jit/metainterp/optimizeopt/test/test_zdisable_opts.py index a71d5bbfd2..9742fe3a6b 100644 --- a/rpython/jit/metainterp/optimizeopt/test/test_zdisable_opts.py +++ b/rpython/jit/metainterp/optimizeopt/test/test_zdisable_opts.py @@ -13,7 +13,7 @@ for optnum in range(len(allopts)): def optimize_loop(self, ops, expected, expected_preamble=None, call_pure_results=None, expected_short=None): - loop = self.parse(ops) + loop = self.parse(ops, postprocess=self.postprocess) if expected != "crash!": expected = self.parse(expected) if expected_preamble: diff --git a/rpython/jit/metainterp/optimizeopt/unroll.py b/rpython/jit/metainterp/optimizeopt/unroll.py index 13c8d78649..1a429a474a 100644 --- a/rpython/jit/metainterp/optimizeopt/unroll.py +++ b/rpython/jit/metainterp/optimizeopt/unroll.py @@ -1,6 +1,5 @@ import sys -from rpython.jit.metainterp.compile import ResumeGuardDescr from rpython.jit.metainterp.history import TargetToken, JitCellToken, Const from rpython.jit.metainterp.inliner import Inliner from rpython.jit.metainterp.optimize import InvalidLoop @@ -8,17 +7,18 @@ from rpython.jit.metainterp.optimizeopt.generalize import KillHugeIntBounds from rpython.jit.metainterp.optimizeopt.optimizer import Optimizer, Optimization from rpython.jit.metainterp.optimizeopt.virtualstate import (VirtualStateConstructor, ShortBoxes, BadVirtualState, VirtualStatesCantMatch) -from rpython.jit.metainterp.resoperation import rop, ResOperation +from rpython.jit.metainterp.resoperation import rop, ResOperation, GuardResOp from rpython.jit.metainterp.resume import Snapshot +from rpython.jit.metainterp import compile from rpython.rlib.debug import debug_print, debug_start, debug_stop # FIXME: Introduce some VirtualOptimizer super class instead -def optimize_unroll(metainterp_sd, loop, optimizations, +def optimize_unroll(metainterp_sd, jitdriver_sd, loop, optimizations, inline_short_preamble=True, start_state=None, export_state=True): - opt = UnrollOptimizer(metainterp_sd, loop, optimizations) + opt = UnrollOptimizer(metainterp_sd, jitdriver_sd, loop, optimizations) opt.inline_short_preamble = inline_short_preamble return opt.propagate_all_forward(start_state, export_state) @@ -52,8 +52,9 @@ class UnrollOptimizer(Optimization): inline_short_preamble = True - def __init__(self, metainterp_sd, loop, optimizations): - self.optimizer = UnrollableOptimizer(metainterp_sd, loop, optimizations) + def __init__(self, metainterp_sd, jitdriver_sd, loop, optimizations): + self.optimizer = UnrollableOptimizer(metainterp_sd, jitdriver_sd, + loop, optimizations) self.boxes_created_this_iteration = None def get_virtual_state(self, args): @@ -72,6 +73,7 @@ class UnrollOptimizer(Optimization): return Snapshot(prev, new_snapshot_args) def propagate_all_forward(self, starting_state, export_state=True): + self.optimizer.exporting_state = export_state loop = self.optimizer.loop self.optimizer.clear_newoperations() @@ -134,6 +136,7 @@ class UnrollOptimizer(Optimization): else: debug_print("Retrace count reached, jumping to preamble") assert cell_token.target_tokens[0].virtual_state is None + jumpop = jumpop.clone() jumpop.setdescr(cell_token.target_tokens[0]) self.optimizer.send_extra_operation(jumpop) return @@ -232,11 +235,7 @@ class UnrollOptimizer(Optimization): self.short_boxes = exported_state.short_boxes self.initial_virtual_state = target_token.virtual_state - seen = {} for box in self.inputargs: - if box in seen: - continue - seen[box] = None preamble_value = exported_state.exported_values[box] value = self.optimizer.getvalue(box) value.import_from(preamble_value, self.optimizer) @@ -506,6 +505,7 @@ class UnrollOptimizer(Optimization): self.import_box(a, inputargs, short_jumpargs, jumpargs) def jump_to_already_compiled_trace(self, jumpop, patchguardop): + jumpop = jumpop.clone() assert jumpop.getopnum() == rop.JUMP cell_token = jumpop.getdescr() @@ -560,8 +560,11 @@ class UnrollOptimizer(Optimization): for guard in extra_guards: if guard.is_guard(): - descr = patchguardop.getdescr().clone_if_mutable() - guard.setdescr(descr) + assert isinstance(patchguardop, GuardResOp) + assert isinstance(guard, GuardResOp) + guard.rd_snapshot = patchguardop.rd_snapshot + guard.rd_frame_info_list = patchguardop.rd_frame_info_list + guard.setdescr(compile.ResumeAtPositionDescr()) self.optimizer.send_extra_operation(guard) try: @@ -590,8 +593,11 @@ class UnrollOptimizer(Optimization): if newop.is_guard(): if not patchguardop: raise InvalidLoop("would like to have short preamble, but it has a guard and there's no guard_future_condition") - descr = patchguardop.getdescr().clone_if_mutable() - newop.setdescr(descr) + assert isinstance(newop, GuardResOp) + assert isinstance(patchguardop, GuardResOp) + newop.rd_snapshot = patchguardop.rd_snapshot + newop.rd_frame_info_list = patchguardop.rd_frame_info_list + newop.setdescr(compile.ResumeAtPositionDescr()) self.optimizer.send_extra_operation(newop) if shop.result in assumed_classes: classbox = self.getvalue(newop.result).get_constant_class(self.optimizer.cpu) diff --git a/rpython/jit/metainterp/optimizeopt/util.py b/rpython/jit/metainterp/optimizeopt/util.py index c99b9f7784..5e209e8fb4 100644 --- a/rpython/jit/metainterp/optimizeopt/util.py +++ b/rpython/jit/metainterp/optimizeopt/util.py @@ -164,7 +164,7 @@ def equaloplists(oplist1, oplist2, strict_fail_args=True, remap={}, assert op1.result.same_box(remap[op2.result]) else: remap[op2.result] = op1.result - if op1.getopnum() not in (rop.JUMP, rop.LABEL): # xxx obscure + if op1.getopnum() not in [rop.JUMP, rop.LABEL] and not op1.is_guard(): assert op1.getdescr() == op2.getdescr() if op1.getfailargs() or op2.getfailargs(): assert len(op1.getfailargs()) == len(op2.getfailargs()) diff --git a/rpython/jit/metainterp/optimizeopt/virtualize.py b/rpython/jit/metainterp/optimizeopt/virtualize.py index 7bced8a664..47a0a2ce86 100644 --- a/rpython/jit/metainterp/optimizeopt/virtualize.py +++ b/rpython/jit/metainterp/optimizeopt/virtualize.py @@ -16,7 +16,7 @@ from rpython.jit.metainterp.optimizeopt.intutils import IntUnbounded class AbstractVirtualValue(optimizer.PtrOptValue): _attrs_ = ('keybox', 'source_op', '_cached_vinfo') box = None - level = optimizer.LEVEL_NONNULL + _tag = optimizer.LEVEL_NONNULL is_about_raw = False _cached_vinfo = None @@ -198,7 +198,7 @@ class AbstractVirtualStructValue(AbstractVirtualValue): fieldvalue.visitor_walk_recursive(visitor) class VirtualValue(AbstractVirtualStructValue): - level = optimizer.LEVEL_KNOWNCLASS + _tag = optimizer.LEVEL_KNOWNCLASS def __init__(self, cpu, known_class, keybox, source_op=None): AbstractVirtualStructValue.__init__(self, cpu, keybox, source_op) diff --git a/rpython/jit/metainterp/optimizeopt/virtualstate.py b/rpython/jit/metainterp/optimizeopt/virtualstate.py index 14cbd7ce4c..2a5b28613e 100644 --- a/rpython/jit/metainterp/optimizeopt/virtualstate.py +++ b/rpython/jit/metainterp/optimizeopt/virtualstate.py @@ -281,7 +281,7 @@ class NotVirtualStateInfo(AbstractVirtualStateInfo): def __init__(self, value, is_opaque=False): self.is_opaque = is_opaque self.known_class = value.get_known_class() - self.level = value.level + self.level = value.getlevel() if value.getintbound() is None: self.intbound = IntUnbounded() else: @@ -346,9 +346,7 @@ class NotVirtualStateInfo(AbstractVirtualStateInfo): if other.level == LEVEL_UNKNOWN: if (box and box.nonnull() and self.known_class.same_constant(cpu.ts.cls_of_box(box))): - op = ResOperation(rop.GUARD_NONNULL, [box], None) - extra_guards.append(op) - op = ResOperation(rop.GUARD_CLASS, [box, self.known_class], None) + op = ResOperation(rop.GUARD_NONNULL_CLASS, [box, self.known_class], None) extra_guards.append(op) return else: diff --git a/rpython/jit/metainterp/pyjitpl.py b/rpython/jit/metainterp/pyjitpl.py index 8d43329e9f..bbc405dba5 100644 --- a/rpython/jit/metainterp/pyjitpl.py +++ b/rpython/jit/metainterp/pyjitpl.py @@ -12,7 +12,7 @@ from rpython.jit.metainterp.history import (Const, ConstInt, ConstPtr, from rpython.jit.metainterp.jitprof import EmptyProfiler from rpython.jit.metainterp.logger import Logger from rpython.jit.metainterp.optimizeopt.util import args_dict -from rpython.jit.metainterp.resoperation import rop +from rpython.jit.metainterp.resoperation import rop, GuardResOp from rpython.rlib import nonconst, rstack from rpython.rlib.debug import debug_start, debug_stop, debug_print from rpython.rlib.debug import have_debug_prints, make_sure_not_resized @@ -1874,23 +1874,15 @@ class MetaInterp(object): moreargs = [box] + extraargs else: moreargs = list(extraargs) - if opnum == rop.GUARD_NOT_FORCED or opnum == rop.GUARD_NOT_FORCED_2: - resumedescr = compile.ResumeGuardForcedDescr(self.staticdata, - self.jitdriver_sd) - elif opnum == rop.GUARD_NOT_INVALIDATED: - resumedescr = compile.ResumeGuardNotInvalidated() - elif opnum == rop.GUARD_FUTURE_CONDITION: - resumedescr = compile.ResumeAtPositionDescr() - else: - resumedescr = compile.ResumeGuardDescr() - guard_op = self.history.record(opnum, moreargs, None, descr=resumedescr) - self.capture_resumedata(resumedescr, resumepc) + guard_op = self.history.record(opnum, moreargs, None) + assert isinstance(guard_op, GuardResOp) + self.capture_resumedata(guard_op, resumepc) self.staticdata.profiler.count_ops(opnum, Counters.GUARDS) # count self.attach_debug_info(guard_op) return guard_op - def capture_resumedata(self, resumedescr, resumepc=-1): + def capture_resumedata(self, guard_op, resumepc=-1): virtualizable_boxes = None if (self.jitdriver_sd.virtualizable_info is not None or self.jitdriver_sd.greenfield_info is not None): @@ -1902,7 +1894,7 @@ class MetaInterp(object): if resumepc >= 0: frame.pc = resumepc resume.capture_resumedata(self.framestack, virtualizable_boxes, - self.virtualref_boxes, resumedescr) + self.virtualref_boxes, guard_op) if self.framestack: self.framestack[-1].pc = saved_pc @@ -2147,7 +2139,9 @@ class MetaInterp(object): assert isinstance(key, compile.ResumeGuardDescr) # store the resumekey.wref_original_loop_token() on 'self' to make # sure that it stays alive as long as this MetaInterp - self.resumekey_original_loop_token = key.wref_original_loop_token() + self.resumekey_original_loop_token = key.rd_loop_token.loop_token_wref() + if self.resumekey_original_loop_token is None: + raise compile.giveup() # should be rare self.staticdata.try_to_free_some_loops() self.initialize_state_from_guard_failure(key, deadframe) try: @@ -2163,13 +2157,8 @@ class MetaInterp(object): self.seen_loop_header_for_jdindex = -1 if isinstance(key, compile.ResumeAtPositionDescr): self.seen_loop_header_for_jdindex = self.jitdriver_sd.index - dont_change_position = True - else: - dont_change_position = False try: - self.prepare_resume_from_failure(key.guard_opnum, - dont_change_position, - deadframe) + self.prepare_resume_from_failure(key.guard_opnum, deadframe) if self.resumekey_original_loop_token is None: # very rare case raise SwitchToBlackhole(Counters.ABORT_BRIDGE) self.interpret() @@ -2312,12 +2301,12 @@ class MetaInterp(object): else: assert 0 self.jitdriver_sd.warmstate.execute_assembler(loop_token, *args) - def prepare_resume_from_failure(self, opnum, dont_change_position, - deadframe): + def prepare_resume_from_failure(self, opnum, deadframe): frame = self.framestack[-1] - if opnum == rop.GUARD_TRUE: # a goto_if_not that jumps only now - if not dont_change_position: - frame.pc = frame.jitcode.follow_jump(frame.pc) + if opnum == rop.GUARD_FUTURE_CONDITION: + pass + elif opnum == rop.GUARD_TRUE: # a goto_if_not that jumps only now + frame.pc = frame.jitcode.follow_jump(frame.pc) elif opnum == rop.GUARD_FALSE: # a goto_if_not that stops jumping; pass # or a switch that was in its "default" case elif opnum == rop.GUARD_VALUE or opnum == rop.GUARD_CLASS: @@ -2341,12 +2330,11 @@ class MetaInterp(object): pass # XXX we want to do something special in resume descr, # but not now elif opnum == rop.GUARD_NO_OVERFLOW: # an overflow now detected - if not dont_change_position: - self.execute_raised(OverflowError(), constant=True) - try: - self.finishframe_exception() - except ChangeFrame: - pass + self.execute_raised(OverflowError(), constant=True) + try: + self.finishframe_exception() + except ChangeFrame: + pass elif opnum == rop.GUARD_OVERFLOW: # no longer overflowing self.clear_exception() else: diff --git a/rpython/jit/metainterp/resoperation.py b/rpython/jit/metainterp/resoperation.py index 185b990b6b..9b322178b2 100644 --- a/rpython/jit/metainterp/resoperation.py +++ b/rpython/jit/metainterp/resoperation.py @@ -91,8 +91,6 @@ class AbstractResOp(object): def clone(self): args = self.getarglist() descr = self.getdescr() - if descr is not None: - descr = descr.clone_if_mutable() op = ResOperation(self.getopnum(), args[:], self.result, descr) if not we_are_translated(): op.name = self.name @@ -217,6 +215,9 @@ class GuardResOp(ResOpWithDescr): _fail_args = None + rd_snapshot = None + rd_frame_info_list = None + def getfailargs(self): return self._fail_args @@ -225,12 +226,18 @@ class GuardResOp(ResOpWithDescr): def copy_and_change(self, opnum, args=None, result=None, descr=None): newop = AbstractResOp.copy_and_change(self, opnum, args, result, descr) + assert isinstance(newop, GuardResOp) newop.setfailargs(self.getfailargs()) + newop.rd_snapshot = self.rd_snapshot + newop.rd_frame_info_list = self.rd_frame_info_list return newop def clone(self): newop = AbstractResOp.clone(self) + assert isinstance(newop, GuardResOp) newop.setfailargs(self.getfailargs()) + newop.rd_snapshot = self.rd_snapshot + newop.rd_frame_info_list = self.rd_frame_info_list return newop # ============ diff --git a/rpython/jit/metainterp/resume.py b/rpython/jit/metainterp/resume.py index fc75b2d3ea..d7fdcc1aa6 100644 --- a/rpython/jit/metainterp/resume.py +++ b/rpython/jit/metainterp/resume.py @@ -55,7 +55,7 @@ def _ensure_parent_resumedata(framestack, n): back.get_list_of_active_boxes(True)) def capture_resumedata(framestack, virtualizable_boxes, virtualref_boxes, - storage): + snapshot_storage): n = len(framestack) - 1 if virtualizable_boxes is not None: boxes = virtualref_boxes + virtualizable_boxes @@ -66,14 +66,14 @@ def capture_resumedata(framestack, virtualizable_boxes, virtualref_boxes, _ensure_parent_resumedata(framestack, n) frame_info_list = FrameInfo(top.parent_resumedata_frame_info_list, top.jitcode, top.pc) - storage.rd_frame_info_list = frame_info_list + snapshot_storage.rd_frame_info_list = frame_info_list snapshot = Snapshot(top.parent_resumedata_snapshot, top.get_list_of_active_boxes(False)) snapshot = Snapshot(snapshot, boxes) - storage.rd_snapshot = snapshot + snapshot_storage.rd_snapshot = snapshot else: - storage.rd_frame_info_list = None - storage.rd_snapshot = Snapshot(None, boxes) + snapshot_storage.rd_frame_info_list = None + snapshot_storage.rd_snapshot = Snapshot(None, boxes) # # The following is equivalent to the RPython-level declaration: @@ -270,8 +270,9 @@ _frame_info_placeholder = (None, 0, 0) class ResumeDataVirtualAdder(VirtualVisitor): - def __init__(self, storage, memo): + def __init__(self, storage, snapshot_storage, memo): self.storage = storage + self.snapshot_storage = snapshot_storage self.memo = memo def make_virtual_info(self, value, fieldnums): @@ -357,13 +358,14 @@ class ResumeDataVirtualAdder(VirtualVisitor): storage = self.storage # make sure that nobody attached resume data to this guard yet assert not storage.rd_numb - snapshot = storage.rd_snapshot + snapshot = self.snapshot_storage.rd_snapshot assert snapshot is not None # is that true? numb, liveboxes_from_env, v = self.memo.number(optimizer, snapshot) self.liveboxes_from_env = liveboxes_from_env self.liveboxes = {} storage.rd_numb = numb - storage.rd_snapshot = None + self.snapshot_storage.rd_snapshot = None + storage.rd_frame_info_list = self.snapshot_storage.rd_frame_info_list # collect liveboxes and virtuals n = len(liveboxes_from_env) - v diff --git a/rpython/jit/metainterp/test/test_ajit.py b/rpython/jit/metainterp/test/test_ajit.py index aeba722644..7982723ba0 100644 --- a/rpython/jit/metainterp/test/test_ajit.py +++ b/rpython/jit/metainterp/test/test_ajit.py @@ -2764,7 +2764,8 @@ class BasicTests: return i # seen = [] - def my_optimize_trace(metainterp_sd, loop, enable_opts, *args, **kwds): + def my_optimize_trace(metainterp_sd, jitdriver_sd, loop, enable_opts, + *args, **kwds): seen.append('unroll' in enable_opts) raise InvalidLoop old_optimize_trace = optimizeopt.optimize_trace diff --git a/rpython/jit/metainterp/test/test_compile.py b/rpython/jit/metainterp/test/test_compile.py index b5082176f6..89eb1f1043 100644 --- a/rpython/jit/metainterp/test/test_compile.py +++ b/rpython/jit/metainterp/test/test_compile.py @@ -10,6 +10,9 @@ from rpython.jit.tool.oparser import parse from rpython.jit.metainterp.optimizeopt import ALL_OPTS_DICT class FakeCPU(object): + class Storage: + pass + class tracker: pass @@ -18,6 +21,7 @@ class FakeCPU(object): self.seen = [] def compile_loop(self, inputargs, operations, token, log=True, name='', logger=None): + token.compiled_loop_token = self.Storage() self.seen.append((inputargs, operations, token)) class FakeLogger(object): diff --git a/rpython/jit/metainterp/test/test_resume.py b/rpython/jit/metainterp/test/test_resume.py index 6647aa6c08..7392a760f4 100644 --- a/rpython/jit/metainterp/test/test_resume.py +++ b/rpython/jit/metainterp/test/test_resume.py @@ -123,7 +123,7 @@ def test_reuse_vinfo(): class FakeVirtualValue(AbstractVirtualValue): def visitor_dispatch_virtual_type(self, *args): return FakeVInfo() - modifier = ResumeDataVirtualAdder(None, None) + modifier = ResumeDataVirtualAdder(None, None, None) v1 = FakeVirtualValue(None, None) vinfo1 = modifier.make_virtual_info(v1, [1, 2, 4]) vinfo2 = modifier.make_virtual_info(v1, [1, 2, 4]) @@ -618,7 +618,7 @@ def test_rebuild_from_resumedata(): FakeFrame("code2", 9, c3, b2)] capture_resumedata(fs, None, [], storage) memo = ResumeDataLoopMemo(FakeMetaInterpStaticData()) - modifier = ResumeDataVirtualAdder(storage, memo) + modifier = ResumeDataVirtualAdder(storage, storage, memo) liveboxes = modifier.finish(FakeOptimizer({})) metainterp = MyMetaInterp() @@ -1022,7 +1022,7 @@ def test_ResumeDataLoopMemo_number_virtuals(): def test_register_virtual_fields(): b1, b2 = BoxInt(), BoxInt() vbox = BoxPtr() - modifier = ResumeDataVirtualAdder(None, None) + modifier = ResumeDataVirtualAdder(None, None, None) modifier.liveboxes_from_env = {} modifier.liveboxes = {} modifier.vfieldboxes = {} @@ -1031,7 +1031,7 @@ def test_register_virtual_fields(): b2: UNASSIGNED} assert modifier.vfieldboxes == {vbox: [b1, b2]} - modifier = ResumeDataVirtualAdder(None, None) + modifier = ResumeDataVirtualAdder(None, None, None) modifier.liveboxes_from_env = {vbox: tag(0, TAGVIRTUAL)} modifier.liveboxes = {} modifier.vfieldboxes = {} @@ -1061,7 +1061,7 @@ def test_virtual_adder_int_constants(): b1s, b2s, b3s = [ConstInt(sys.maxint), ConstInt(2**16), ConstInt(-65)] storage = make_storage(b1s, b2s, b3s) memo = ResumeDataLoopMemo(FakeMetaInterpStaticData()) - modifier = ResumeDataVirtualAdder(storage, memo) + modifier = ResumeDataVirtualAdder(storage, storage, memo) liveboxes = modifier.finish(FakeOptimizer({})) assert storage.rd_snapshot is None cpu = MyCPU([]) @@ -1075,14 +1075,14 @@ def test_virtual_adder_memo_const_sharing(): b1s, b2s, b3s = [ConstInt(sys.maxint), ConstInt(2**16), ConstInt(-65)] storage = make_storage(b1s, b2s, b3s) memo = ResumeDataLoopMemo(FakeMetaInterpStaticData()) - modifier = ResumeDataVirtualAdder(storage, memo) + modifier = ResumeDataVirtualAdder(storage, storage, memo) modifier.finish(FakeOptimizer({})) assert len(memo.consts) == 2 assert storage.rd_consts is memo.consts b1s, b2s, b3s = [ConstInt(sys.maxint), ConstInt(2**17), ConstInt(-65)] storage2 = make_storage(b1s, b2s, b3s) - modifier2 = ResumeDataVirtualAdder(storage2, memo) + modifier2 = ResumeDataVirtualAdder(storage2, storage2, memo) modifier2.finish(FakeOptimizer({})) assert len(memo.consts) == 3 assert storage2.rd_consts is memo.consts @@ -1137,7 +1137,7 @@ def test_virtual_adder_no_op_renaming(): b1s, b2s, b3s = [BoxInt(1), BoxInt(2), BoxInt(3)] storage = make_storage(b1s, b2s, b3s) memo = ResumeDataLoopMemo(FakeMetaInterpStaticData()) - modifier = ResumeDataVirtualAdder(storage, memo) + modifier = ResumeDataVirtualAdder(storage, storage, memo) b1_2 = BoxInt() class FakeValue(object): @@ -1169,7 +1169,7 @@ def test_virtual_adder_make_constant(): b1s = ConstInt(111) storage = make_storage(b1s, b2s, b3s) memo = ResumeDataLoopMemo(FakeMetaInterpStaticData()) - modifier = ResumeDataVirtualAdder(storage, memo) + modifier = ResumeDataVirtualAdder(storage, storage, memo) liveboxes = modifier.finish(FakeOptimizer({})) b2t, b3t = [BoxPtr(demo55o), BoxInt(33)] newboxes = _resume_remap(liveboxes, [b2s, b3s], b2t, b3t) @@ -1190,7 +1190,7 @@ def test_virtual_adder_make_virtual(): c1s = ConstInt(111) storage = Storage() memo = ResumeDataLoopMemo(FakeMetaInterpStaticData()) - modifier = ResumeDataVirtualAdder(storage, memo) + modifier = ResumeDataVirtualAdder(storage, storage, memo) modifier.liveboxes_from_env = {} modifier.liveboxes = {} modifier.vfieldboxes = {} @@ -1273,7 +1273,7 @@ def test_virtual_adder_make_varray(): c1s = ConstInt(111) storage = Storage() memo = ResumeDataLoopMemo(FakeMetaInterpStaticData()) - modifier = ResumeDataVirtualAdder(storage, memo) + modifier = ResumeDataVirtualAdder(storage, storage, memo) modifier.liveboxes_from_env = {} modifier.liveboxes = {} modifier.vfieldboxes = {} @@ -1321,7 +1321,7 @@ def test_virtual_adder_make_vstruct(): c1s = ConstInt(111) storage = Storage() memo = ResumeDataLoopMemo(FakeMetaInterpStaticData()) - modifier = ResumeDataVirtualAdder(storage, memo) + modifier = ResumeDataVirtualAdder(storage, storage, memo) modifier.liveboxes_from_env = {} modifier.liveboxes = {} modifier.vfieldboxes = {} @@ -1364,7 +1364,7 @@ def test_virtual_adder_pending_fields(): b2s, b4s = [BoxPtr(), BoxPtr()] storage = Storage() memo = ResumeDataLoopMemo(FakeMetaInterpStaticData()) - modifier = ResumeDataVirtualAdder(storage, memo) + modifier = ResumeDataVirtualAdder(storage, storage, memo) modifier.liveboxes_from_env = {} modifier.liveboxes = {} modifier.vfieldboxes = {} @@ -1403,7 +1403,7 @@ def test_virtual_adder_pending_fields_and_arrayitems(): class Storage(object): pass storage = Storage() - modifier = ResumeDataVirtualAdder(storage, None) + modifier = ResumeDataVirtualAdder(storage, storage, None) modifier._add_pending_fields([]) assert not storage.rd_pendingfields # @@ -1411,7 +1411,7 @@ def test_virtual_adder_pending_fields_and_arrayitems(): pass field_a = FieldDescr() storage = Storage() - modifier = ResumeDataVirtualAdder(storage, None) + modifier = ResumeDataVirtualAdder(storage, storage, None) modifier.liveboxes_from_env = {42: rffi.cast(rffi.SHORT, 1042), 61: rffi.cast(rffi.SHORT, 1061)} modifier._add_pending_fields([(field_a, 42, 61, -1)]) @@ -1425,7 +1425,7 @@ def test_virtual_adder_pending_fields_and_arrayitems(): # array_a = FieldDescr() storage = Storage() - modifier = ResumeDataVirtualAdder(storage, None) + modifier = ResumeDataVirtualAdder(storage, storage, None) modifier.liveboxes_from_env = {42: rffi.cast(rffi.SHORT, 1042), 61: rffi.cast(rffi.SHORT, 1061), 62: rffi.cast(rffi.SHORT, 1062), @@ -1506,7 +1506,7 @@ def test_invalidation_needed(): metainterp_sd = FakeMetaInterpStaticData() metainterp_sd.options = options memo = ResumeDataLoopMemo(metainterp_sd) - modifier = ResumeDataVirtualAdder(None, memo) + modifier = ResumeDataVirtualAdder(None, None, memo) for i in range(5): assert not modifier._invalidation_needed(5, i) diff --git a/rpython/jit/metainterp/virtualizable.py b/rpython/jit/metainterp/virtualizable.py index bad819a98e..e334285077 100644 --- a/rpython/jit/metainterp/virtualizable.py +++ b/rpython/jit/metainterp/virtualizable.py @@ -49,7 +49,13 @@ class VirtualizableInfo(object): ARRAYPTR = fieldType(VTYPE, name) ARRAY = deref(ARRAYPTR) assert isinstance(ARRAYPTR, lltype.Ptr) - assert isinstance(ARRAY, lltype.GcArray) + if not isinstance(ARRAY, lltype.GcArray): + raise Exception( + "The virtualizable field '%s' is not an array (found %r)." + " It usually means that you must try harder to ensure that" + " the list is not resized at run-time. You can do that by" + " using rpython.rlib.debug.make_sure_not_resized()." % + (name, ARRAY)) ARRAYITEMTYPES.append(arrayItem(ARRAY)) self.array_descrs = [cpu.arraydescrof(deref(fieldType(VTYPE, name))) for name in array_fields] diff --git a/rpython/jit/tool/oparser.py b/rpython/jit/tool/oparser.py index fa09d1995f..9a0099c74e 100644 --- a/rpython/jit/tool/oparser.py +++ b/rpython/jit/tool/oparser.py @@ -61,9 +61,10 @@ class OpParser(object): def __init__(self, input, cpu, namespace, type_system, boxkinds, invent_fail_descr=default_fail_descr, - nonstrict=False): + nonstrict=False, postproces=None): self.input = input self.vars = {} + self._postproces = postproces self.cpu = cpu self._consts = namespace self.type_system = type_system @@ -237,8 +238,6 @@ class OpParser(object): fail_args.append(fail_arg) if descr is None and self.invent_fail_descr: descr = self.invent_fail_descr(self.model, opnum, fail_args) - if hasattr(descr, '_oparser_uses_descr_of_guard'): - descr._oparser_uses_descr_of_guard(self, fail_args) else: fail_args = None if opnum == rop.FINISH: @@ -250,7 +249,7 @@ class OpParser(object): return opnum, args, descr, fail_args - def create_op(self, opnum, args, result, descr): + def create_op(self, opnum, args, result, descr, fail_args): if opnum == ESCAPE_OP.OPNUM: op = ESCAPE_OP(result) op.initarglist(args) @@ -262,7 +261,12 @@ class OpParser(object): assert descr is None return op else: - return ResOperation(opnum, args, result, descr) + res = ResOperation(opnum, args, result, descr) + if fail_args is not None: + res.setfailargs(fail_args) + if self._postproces: + self._postproces(res) + return res def parse_result_op(self, line): res, op = line.split("=", 1) @@ -273,16 +277,12 @@ class OpParser(object): raise ParseError("Double assign to var %s in line: %s" % (res, line)) rvar = self.box_for_var(res) self.vars[res] = rvar - res = self.create_op(opnum, args, rvar, descr) - if fail_args is not None: - res.setfailargs(fail_args) + res = self.create_op(opnum, args, rvar, descr, fail_args) return res def parse_op_no_result(self, line): opnum, args, descr, fail_args = self.parse_op(line) - res = self.create_op(opnum, args, None, descr) - if fail_args is not None: - res.setfailargs(fail_args) + res = self.create_op(opnum, args, None, descr, fail_args) return res def parse_next_op(self, line): @@ -377,11 +377,12 @@ class OpParser(object): def parse(input, cpu=None, namespace=None, type_system='lltype', boxkinds=None, invent_fail_descr=default_fail_descr, - no_namespace=False, nonstrict=False, OpParser=OpParser): + no_namespace=False, nonstrict=False, OpParser=OpParser, + postprocess=None): if namespace is None and not no_namespace: namespace = {} return OpParser(input, cpu, namespace, type_system, boxkinds, - invent_fail_descr, nonstrict).parse() + invent_fail_descr, nonstrict, postprocess).parse() def pure_parse(*args, **kwds): kwds['invent_fail_descr'] = None diff --git a/rpython/memory/gc/base.py b/rpython/memory/gc/base.py index 560f267efa..dc24149a4c 100644 --- a/rpython/memory/gc/base.py +++ b/rpython/memory/gc/base.py @@ -219,19 +219,42 @@ class GCBase(object): def _trace_slow_path(self, obj, callback, arg): typeid = self.get_type_id(obj) if self.has_gcptr_in_varsize(typeid): - item = obj + self.varsize_offset_to_variable_part(typeid) length = (obj + self.varsize_offset_to_length(typeid)).signed[0] - offsets = self.varsize_offsets_to_gcpointers_in_var_part(typeid) - itemlength = self.varsize_item_sizes(typeid) - while length > 0: - j = 0 - while j < len(offsets): - itemobj = item + offsets[j] - if self.points_to_valid_gc_object(itemobj): - callback(itemobj, arg) - j += 1 - item += itemlength - length -= 1 + if length > 0: + item = obj + self.varsize_offset_to_variable_part(typeid) + offsets = self.varsize_offsets_to_gcpointers_in_var_part(typeid) + itemlength = self.varsize_item_sizes(typeid) + len_offsets = len(offsets) + if len_offsets == 1: # common path #1 + offsets0 = offsets[0] + while length > 0: + itemobj0 = item + offsets0 + if self.points_to_valid_gc_object(itemobj0): + callback(itemobj0, arg) + item += itemlength + length -= 1 + elif len_offsets == 2: # common path #2 + offsets0 = offsets[0] + offsets1 = offsets[1] + while length > 0: + itemobj0 = item + offsets0 + if self.points_to_valid_gc_object(itemobj0): + callback(itemobj0, arg) + itemobj1 = item + offsets1 + if self.points_to_valid_gc_object(itemobj1): + callback(itemobj1, arg) + item += itemlength + length -= 1 + else: # general path + while length > 0: + j = 0 + while j < len_offsets: + itemobj = item + offsets[j] + if self.points_to_valid_gc_object(itemobj): + callback(itemobj, arg) + j += 1 + item += itemlength + length -= 1 if self.has_custom_trace(typeid): self.custom_trace_dispatcher(obj, typeid, callback, arg) _trace_slow_path._annspecialcase_ = 'specialize:arg(2)' diff --git a/rpython/memory/gc/incminimark.py b/rpython/memory/gc/incminimark.py index fb00eaaa32..de643485e3 100644 --- a/rpython/memory/gc/incminimark.py +++ b/rpython/memory/gc/incminimark.py @@ -773,11 +773,17 @@ class IncrementalMiniMarkGC(MovingGCBase): raise MemoryError # # If somebody calls this function a lot, we must eventually - # force a full collection. XXX make this more incremental! - if (float(self.get_total_memory_used()) + raw_malloc_usage(totalsize) > - self.next_major_collection_threshold): - self.gc_step_until(STATE_SWEEPING) - self.gc_step_until(STATE_FINALIZING, raw_malloc_usage(totalsize)) + # force a collection. XXX make this more incremental! For now + # the logic is to first do a minor GC only, and check if that + # was enough to free a bunch of large young objects. If not, + # we do a complete major GC. + if self.get_total_memory_free() < raw_malloc_usage(totalsize): + self.minor_collection() + if self.get_total_memory_free() < (raw_malloc_usage(totalsize) + + self.nursery_size // 2): + self.gc_step_until(STATE_SWEEPING) + self.gc_step_until(STATE_FINALIZING, + raw_malloc_usage(totalsize)) # # Check if the object would fit in the ArenaCollection. if raw_malloc_usage(totalsize) <= self.small_request_threshold: @@ -1054,6 +1060,10 @@ class IncrementalMiniMarkGC(MovingGCBase): """ return self.ac.total_memory_used + self.rawmalloced_total_size + def get_total_memory_free(self): + return (self.next_major_collection_threshold - + float(self.get_total_memory_used())) + def card_marking_words_for_length(self, length): # --- Unoptimized version: #num_bits = ((length-1) >> self.card_page_shift) + 1 diff --git a/rpython/memory/gc/inspector.py b/rpython/memory/gc/inspector.py index f6adf00400..7ddd0ff32c 100644 --- a/rpython/memory/gc/inspector.py +++ b/rpython/memory/gc/inspector.py @@ -1,7 +1,7 @@ """ Utility RPython functions to inspect objects in the GC. """ -from rpython.rtyper.lltypesystem import lltype, llmemory, rffi +from rpython.rtyper.lltypesystem import lltype, llmemory, rffi, llgroup from rpython.rlib.objectmodel import free_non_gc_object from rpython.rtyper.module.ll_os import UNDERSCORE_ON_WIN32 from rpython.rlib import rposix, rgc @@ -238,3 +238,8 @@ def dump_rpy_heap(gc, fd): def get_typeids_z(gc): srcaddress = gc.root_walker.gcdata.typeids_z return llmemory.cast_adr_to_ptr(srcaddress, lltype.Ptr(rgc.ARRAY_OF_CHAR)) + +def get_typeids_list(gc): + srcaddress = gc.root_walker.gcdata.typeids_list + return llmemory.cast_adr_to_ptr(srcaddress, lltype.Ptr(ARRAY_OF_HALFWORDS)) +ARRAY_OF_HALFWORDS = lltype.Array(llgroup.HALFWORD) diff --git a/rpython/memory/gctransform/framework.py b/rpython/memory/gctransform/framework.py index 2651af50b4..8b0bd12056 100644 --- a/rpython/memory/gctransform/framework.py +++ b/rpython/memory/gctransform/framework.py @@ -170,6 +170,7 @@ class BaseFrameworkGCTransformer(GCTransformer): gcdata.static_root_end = a_random_address # patched in finish() gcdata.max_type_id = 13 # patched in finish() gcdata.typeids_z = a_random_address # patched in finish() + gcdata.typeids_list = a_random_address # patched in finish() self.gcdata = gcdata self.malloc_fnptr_cache = {} @@ -205,6 +206,7 @@ class BaseFrameworkGCTransformer(GCTransformer): data_classdef.generalize_attr('static_root_end', SomeAddress()) data_classdef.generalize_attr('max_type_id', annmodel.SomeInteger()) data_classdef.generalize_attr('typeids_z', SomeAddress()) + data_classdef.generalize_attr('typeids_list', SomeAddress()) annhelper = annlowlevel.MixLevelHelperAnnotator(self.translator.rtyper) @@ -455,6 +457,11 @@ class BaseFrameworkGCTransformer(GCTransformer): [s_gc], SomePtr(lltype.Ptr(rgc.ARRAY_OF_CHAR)), minimal_transform=False) + self.get_typeids_list_ptr = getfn(inspector.get_typeids_list, + [s_gc], + SomePtr(lltype.Ptr( + lltype.Array(llgroup.HALFWORD))), + minimal_transform=False) self.set_max_heap_size_ptr = getfn(GCClass.set_max_heap_size.im_func, [s_gc, @@ -596,7 +603,8 @@ class BaseFrameworkGCTransformer(GCTransformer): newgcdependencies = [] newgcdependencies.append(ll_static_roots_inside) ll_instance.inst_max_type_id = len(group.members) - typeids_z = self.write_typeid_list() + # + typeids_z, typeids_list = self.write_typeid_list() ll_typeids_z = lltype.malloc(rgc.ARRAY_OF_CHAR, len(typeids_z), immortal=True) @@ -604,6 +612,15 @@ class BaseFrameworkGCTransformer(GCTransformer): ll_typeids_z[i] = typeids_z[i] ll_instance.inst_typeids_z = llmemory.cast_ptr_to_adr(ll_typeids_z) newgcdependencies.append(ll_typeids_z) + # + ll_typeids_list = lltype.malloc(lltype.Array(llgroup.HALFWORD), + len(typeids_list), + immortal=True) + for i in range(len(typeids_list)): + ll_typeids_list[i] = typeids_list[i] + ll_instance.inst_typeids_list= llmemory.cast_ptr_to_adr(ll_typeids_list) + newgcdependencies.append(ll_typeids_list) + # return newgcdependencies def get_finish_tables(self): @@ -624,6 +641,13 @@ class BaseFrameworkGCTransformer(GCTransformer): # XXX argh argh, this only gives the member index but not the # real typeid, which is a complete mess to obtain now... all_ids = self.layoutbuilder.id_of_type.items() + list_data = [] + ZERO = rffi.cast(llgroup.HALFWORD, 0) + for _, typeinfo in all_ids: + while len(list_data) <= typeinfo.index: + list_data.append(ZERO) + list_data[typeinfo.index] = typeinfo + # all_ids = [(typeinfo.index, TYPE) for (TYPE, typeinfo) in all_ids] all_ids = dict(all_ids) f = udir.join("typeids.txt").open("w") @@ -632,9 +656,10 @@ class BaseFrameworkGCTransformer(GCTransformer): f.close() try: import zlib - return zlib.compress(udir.join("typeids.txt").read(), 9) + z_data = zlib.compress(udir.join("typeids.txt").read(), 9) except ImportError: - return '' + z_data = '' + return z_data, list_data def transform_graph(self, graph): func = getattr(graph, 'func', None) @@ -1175,6 +1200,13 @@ class BaseFrameworkGCTransformer(GCTransformer): resultvar=hop.spaceop.result) self.pop_roots(hop, livevars) + def gct_gc_typeids_list(self, hop): + livevars = self.push_roots(hop) + hop.genop("direct_call", + [self.get_typeids_list_ptr, self.c_const_gc], + resultvar=hop.spaceop.result) + self.pop_roots(hop, livevars) + def _set_into_gc_array_part(self, op): if op.opname == 'setarrayitem': return op.args[1] diff --git a/rpython/memory/test/gc_test_base.py b/rpython/memory/test/gc_test_base.py index b28df2d214..984a16e591 100644 --- a/rpython/memory/test/gc_test_base.py +++ b/rpython/memory/test/gc_test_base.py @@ -607,6 +607,58 @@ class GCTest(object): return rgc.can_move(lltype.malloc(TP, 1)) assert self.interpret(func, []) == self.GC_CAN_MOVE + def test_trace_array_of_structs(self): + R = lltype.GcStruct('R', ('i', lltype.Signed)) + S1 = lltype.GcArray(('p1', lltype.Ptr(R))) + S2 = lltype.GcArray(('p1', lltype.Ptr(R)), + ('p2', lltype.Ptr(R))) + S3 = lltype.GcArray(('p1', lltype.Ptr(R)), + ('p2', lltype.Ptr(R)), + ('p3', lltype.Ptr(R))) + def func(): + s1 = lltype.malloc(S1, 2) + s1[0].p1 = lltype.malloc(R) + s1[1].p1 = lltype.malloc(R) + s2 = lltype.malloc(S2, 2) + s2[0].p1 = lltype.malloc(R) + s2[0].p2 = lltype.malloc(R) + s2[1].p1 = lltype.malloc(R) + s2[1].p2 = lltype.malloc(R) + s3 = lltype.malloc(S3, 2) + s3[0].p1 = lltype.malloc(R) + s3[0].p2 = lltype.malloc(R) + s3[0].p3 = lltype.malloc(R) + s3[1].p1 = lltype.malloc(R) + s3[1].p2 = lltype.malloc(R) + s3[1].p3 = lltype.malloc(R) + s1[0].p1.i = 100 + s1[1].p1.i = 101 + s2[0].p1.i = 102 + s2[0].p2.i = 103 + s2[1].p1.i = 104 + s2[1].p2.i = 105 + s3[0].p1.i = 106 + s3[0].p2.i = 107 + s3[0].p3.i = 108 + s3[1].p1.i = 109 + s3[1].p2.i = 110 + s3[1].p3.i = 111 + rgc.collect() + return ((s1[0].p1.i == 100) + + (s1[1].p1.i == 101) + + (s2[0].p1.i == 102) + + (s2[0].p2.i == 103) + + (s2[1].p1.i == 104) + + (s2[1].p2.i == 105) + + (s3[0].p1.i == 106) + + (s3[0].p2.i == 107) + + (s3[0].p3.i == 108) + + (s3[1].p1.i == 109) + + (s3[1].p2.i == 110) + + (s3[1].p3.i == 111)) + res = self.interpret(func, []) + assert res == 12 + def test_shrink_array(self): from rpython.rtyper.lltypesystem.rstr import STR diff --git a/rpython/rlib/buffer.py b/rpython/rlib/buffer.py index 29820b3226..3fd6d5a017 100644 --- a/rpython/rlib/buffer.py +++ b/rpython/rlib/buffer.py @@ -115,3 +115,8 @@ class SubBuffer(Buffer): return # otherwise, adding self.offset might make 'start' # out of bounds self.buffer.setslice(self.offset + start, string) + + def get_raw_address(self): + from rpython.rtyper.lltypesystem import rffi + ptr = self.buffer.get_raw_address() + return rffi.ptradd(ptr, self.offset) diff --git a/rpython/rlib/rgc.py b/rpython/rlib/rgc.py index 0760fea493..ae13a2cbf4 100644 --- a/rpython/rlib/rgc.py +++ b/rpython/rlib/rgc.py @@ -445,6 +445,10 @@ def get_typeids_z(): "NOT_RPYTHON" raise NotImplementedError +def get_typeids_list(): + "NOT_RPYTHON" + raise NotImplementedError + def has_gcflag_extra(): "NOT_RPYTHON" return True @@ -644,6 +648,18 @@ class Entry(ExtRegistryEntry): return hop.genop('gc_typeids_z', [], resulttype = hop.r_result) class Entry(ExtRegistryEntry): + _about_ = get_typeids_list + + def compute_result_annotation(self): + from rpython.rtyper.llannotation import SomePtr + from rpython.rtyper.lltypesystem import llgroup + return SomePtr(lltype.Ptr(lltype.Array(llgroup.HALFWORD))) + + def specialize_call(self, hop): + hop.exception_is_here() + return hop.genop('gc_typeids_list', [], resulttype = hop.r_result) + +class Entry(ExtRegistryEntry): _about_ = (has_gcflag_extra, get_gcflag_extra, toggle_gcflag_extra) def compute_result_annotation(self, s_arg=None): from rpython.annotator.model import s_Bool diff --git a/rpython/rlib/rtermios.py b/rpython/rlib/rtermios.py index 2fedc8975c..833b1ab34f 100644 --- a/rpython/rlib/rtermios.py +++ b/rpython/rlib/rtermios.py @@ -3,36 +3,178 @@ # returns list of mostly-strings of length one, but with few ints # inside, so we make sure it works -import termios -from termios import * +from rpython.rtyper.lltypesystem import rffi, lltype +from rpython.rtyper.tool import rffi_platform +from rpython.translator.tool.cbuild import ExternalCompilationInfo + +from rpython.rlib import rposix +from rpython.rlib.rarithmetic import intmask + +eci = ExternalCompilationInfo( + includes = ['termios.h', 'unistd.h', 'sys/ioctl.h'] +) + +class CConfig: + _compilation_info_ = eci + _HAVE_STRUCT_TERMIOS_C_ISPEED = rffi_platform.Defined( + '_HAVE_STRUCT_TERMIOS_C_ISPEED') + _HAVE_STRUCT_TERMIOS_C_OSPEED = rffi_platform.Defined( + '_HAVE_STRUCT_TERMIOS_C_OSPEED') + +CONSTANT_NAMES = ( + # cfgetospeed(), cfsetospeed() constants + """B0 B50 B75 B110 B134 B150 B200 B300 B600 B1200 B1800 B2400 B4800 B9600 + B19200 B38400 B57600 B115200 B230400 B460800 CBAUDEX + """ + # tcsetattr() constants + """TCSANOW TCSADRAIN TCSAFLUSH TCSASOFT + """ + # tcflush() constants + """TCIFLUSH TCOFLUSH TCIOFLUSH + """ + # tcflow() constants + """TCOOFF TCOON TCIOFF TCION + """ + # struct termios.c_iflag constants + """IGNBRK BRKINT IGNPAR PARMRK INPCK ISTRIP INLCR IGNCR ICRNL IUCLC + IXON IXANY IXOFF IMAXBEL + """ + # struct termios.c_oflag constants + """OPOST OLCUC ONLCR OCRNL ONOCR ONLRET OFILL OFDEL + NLDLY CRDLY TABDLY BSDLY VTDLY FFDLY + """ + # struct termios.c_oflag-related values (delay mask) + """NL0 NL1 CR0 CR1 CR2 CR3 TAB0 TAB1 TAB2 TAB3 XTABS + BS0 BS1 VT0 VT1 FF0 FF1 + """ + # struct termios.c_cflag constants + """CSIZE CSTOPB CREAD PARENB PARODD HUPCL CLOCAL CIBAUD CRTSCTS + """ + # struct termios.c_cflag-related values (character size) + """CS5 CS6 CS7 CS8 + """ + # struct termios.c_lflag constants + """ISIG ICANON XCASE ECHO ECHOE ECHOK ECHONL ECHOCTL ECHOPRT ECHOKE + FLUSHO NOFLSH TOSTOP PENDIN IEXTEN + """ + # indexes into the control chars array returned by tcgetattr() + """VINTR VQUIT VERASE VKILL VEOF VTIME VMIN VSWTC VSWTCH VSTART VSTOP + VSUSP VEOL VREPRINT VDISCARD VWERASE VLNEXT VEOL2 + """ + # Others? + """CBAUD CDEL CDSUSP CEOF CEOL CEOL2 CEOT CERASE CESC CFLUSH CINTR CKILL + CLNEXT CNUL COMMON CQUIT CRPRNT CSTART CSTOP CSUSP CSWTCH CWERASE + EXTA EXTB + FIOASYNC FIOCLEX FIONBIO FIONCLEX FIONREAD + IBSHIFT INIT_C_CC IOCSIZE_MASK IOCSIZE_SHIFT + NCC NCCS NSWTCH N_MOUSE N_PPP N_SLIP N_STRIP N_TTY + TCFLSH TCGETA TCGETS TCSBRK TCSBRKP TCSETA TCSETAF TCSETAW TCSETS + TCSETSF TCSETSW TCXONC + TIOCCONS TIOCEXCL TIOCGETD TIOCGICOUNT TIOCGLCKTRMIOS TIOCGPGRP + TIOCGSERIAL TIOCGSOFTCAR TIOCGWINSZ TIOCINQ TIOCLINUX TIOCMBIC + TIOCMBIS TIOCMGET TIOCMIWAIT TIOCMSET TIOCM_CAR TIOCM_CD TIOCM_CTS + TIOCM_DSR TIOCM_DTR TIOCM_LE TIOCM_RI TIOCM_RNG TIOCM_RTS TIOCM_SR + TIOCM_ST TIOCNOTTY TIOCNXCL TIOCOUTQ TIOCPKT TIOCPKT_DATA + TIOCPKT_DOSTOP TIOCPKT_FLUSHREAD TIOCPKT_FLUSHWRITE TIOCPKT_NOSTOP + TIOCPKT_START TIOCPKT_STOP TIOCSCTTY TIOCSERCONFIG TIOCSERGETLSR + TIOCSERGETMULTI TIOCSERGSTRUCT TIOCSERGWILD TIOCSERSETMULTI + TIOCSERSWILD TIOCSER_TEMT TIOCSETD TIOCSLCKTRMIOS TIOCSPGRP + TIOCSSERIAL TIOCSSOFTCAR TIOCSTI TIOCSWINSZ TIOCTTYGSTRUCT + """).split() + +for name in CONSTANT_NAMES: + setattr(CConfig, name, rffi_platform.DefinedConstantInteger(name)) + +c_config = rffi_platform.configure(CConfig) + +# Copy VSWTCH to VSWTC and vice-versa +if c_config['VSWTC'] is None: + c_config['VSWTC'] = c_config['VSWTCH'] +if c_config['VSWTCH'] is None: + c_config['VSWTCH'] = c_config['VSWTC'] + +all_constants = {} +for name in CONSTANT_NAMES: + value = c_config[name] + if value is not None: + globals()[name] = value + all_constants[name] = value + +TCFLAG_T = rffi.UINT +CC_T = rffi.UCHAR +SPEED_T = rffi.UINT + +_add = [] +if c_config['_HAVE_STRUCT_TERMIOS_C_ISPEED']: + _add.append(('c_ispeed', SPEED_T)) +if c_config['_HAVE_STRUCT_TERMIOS_C_OSPEED']: + _add.append(('c_ospeed', SPEED_T)) +TERMIOSP = rffi.CStructPtr('termios', ('c_iflag', TCFLAG_T), ('c_oflag', TCFLAG_T), + ('c_cflag', TCFLAG_T), ('c_lflag', TCFLAG_T), + ('c_line', CC_T), + ('c_cc', lltype.FixedSizeArray(CC_T, NCCS)), *_add) + +def c_external(name, args, result): + return rffi.llexternal(name, args, result, compilation_info=eci) + +c_tcgetattr = c_external('tcgetattr', [rffi.INT, TERMIOSP], rffi.INT) +c_tcsetattr = c_external('tcsetattr', [rffi.INT, rffi.INT, TERMIOSP], rffi.INT) +c_cfgetispeed = c_external('cfgetispeed', [TERMIOSP], SPEED_T) +c_cfgetospeed = c_external('cfgetospeed', [TERMIOSP], SPEED_T) +c_cfsetispeed = c_external('cfsetispeed', [TERMIOSP, SPEED_T], rffi.INT) +c_cfsetospeed = c_external('cfsetospeed', [TERMIOSP, SPEED_T], rffi.INT) + +c_tcsendbreak = c_external('tcsendbreak', [rffi.INT, rffi.INT], rffi.INT) +c_tcdrain = c_external('tcdrain', [rffi.INT], rffi.INT) +c_tcflush = c_external('tcflush', [rffi.INT, rffi.INT], rffi.INT) +c_tcflow = c_external('tcflow', [rffi.INT, rffi.INT], rffi.INT) + def tcgetattr(fd): - # NOT_RPYTHON - try: - lst = list(termios.tcgetattr(fd)) - except termios.error, e: - raise OSError(*e.args) - cc = lst[-1] - next_cc = [] - for c in cc: - if isinstance(c, int): - next_cc.append(chr(c)) - else: - next_cc.append(c) - lst[-1] = next_cc - return tuple(lst) - -def tcsetattr(fd, when, mode): - # NOT_RPYTHON - # there are some bizarre requirements for that, stealing directly - # from cpython - mode_l = list(mode) - if mode_l[3] & termios.ICANON: - cc = mode_l[-1] - cc[termios.VMIN] = ord(cc[termios.VMIN]) - cc[termios.VTIME] = ord(cc[termios.VTIME]) - mode_l[-1] = cc - try: - return termios.tcsetattr(fd, when, mode_l) - except termios.error, e: - raise OSError(*e.args) + with lltype.scoped_alloc(TERMIOSP.TO) as c_struct: + if c_tcgetattr(fd, c_struct) < 0: + raise OSError(rposix.get_errno(), 'tcgetattr failed') + cc = [chr(c_struct.c_c_cc[i]) for i in range(NCCS)] + ispeed = c_cfgetispeed(c_struct) + ospeed = c_cfgetospeed(c_struct) + result = (intmask(c_struct.c_c_iflag), intmask(c_struct.c_c_oflag), + intmask(c_struct.c_c_cflag), intmask(c_struct.c_c_lflag), + intmask(ispeed), intmask(ospeed), cc) + return result + + +# This function is not an exact replacement of termios.tcsetattr: +# the last attribute must be a list of chars. +def tcsetattr(fd, when, attributes): + with lltype.scoped_alloc(TERMIOSP.TO) as c_struct: + rffi.setintfield(c_struct, 'c_c_iflag', attributes[0]) + rffi.setintfield(c_struct, 'c_c_oflag', attributes[1]) + rffi.setintfield(c_struct, 'c_c_cflag', attributes[2]) + rffi.setintfield(c_struct, 'c_c_lflag', attributes[3]) + ispeed = attributes[4] + ospeed = attributes[5] + cc = attributes[6] + for i in range(NCCS): + c_struct.c_c_cc[i] = rffi.r_uchar(ord(cc[i][0])) + if c_cfsetispeed(c_struct, ispeed) < 0: + raise OSError(rposix.get_errno(), 'tcsetattr failed') + if c_cfsetospeed(c_struct, ospeed) < 0: + raise OSError(rposix.get_errno(), 'tcsetattr failed') + if c_tcsetattr(fd, when, c_struct) < 0: + raise OSError(rposix.get_errno(), 'tcsetattr failed') + +def tcsendbreak(fd, duration): + if c_tcsendbreak(fd, duration) < 0: + raise OSError(rposix.get_errno(), 'tcsendbreak failed') + +def tcdrain(fd): + if c_tcdrain(fd) < 0: + raise OSError(rposix.get_errno(), 'tcdrain failed') + +def tcflush(fd, queue_selector): + if c_tcflush(fd, queue_selector) < 0: + raise OSError(rposix.get_errno(), 'tcflush failed') + +def tcflow(fd, action): + if c_tcflow(fd, action) < 0: + raise OSError(rposix.get_errno(), 'tcflow failed') diff --git a/rpython/rlib/rwin32.py b/rpython/rlib/rwin32.py index 3653fa5c4d..5d42cc603d 100644 --- a/rpython/rlib/rwin32.py +++ b/rpython/rlib/rwin32.py @@ -118,11 +118,15 @@ if WIN32: INVALID_HANDLE_VALUE = rffi.cast(HANDLE, -1) PFILETIME = rffi.CArrayPtr(FILETIME) - _GetLastError = winexternal('GetLastError', [], DWORD, releasegil=False) - _SetLastError = winexternal('SetLastError', [DWORD], lltype.Void) + _GetLastError = winexternal('GetLastError', [], DWORD, + _nowrapper=True, sandboxsafe=True) + _SetLastError = winexternal('SetLastError', [DWORD], lltype.Void, + _nowrapper=True, sandboxsafe=True) + @jit.dont_look_inside def GetLastError(): return rffi.cast(lltype.Signed, _GetLastError()) + @jit.dont_look_inside def SetLastError(err): _SetLastError(rffi.cast(DWORD, err)) diff --git a/rpython/rlib/rzlib.py b/rpython/rlib/rzlib.py index ae7c31a787..f2970dc6d1 100644 --- a/rpython/rlib/rzlib.py +++ b/rpython/rlib/rzlib.py @@ -167,6 +167,7 @@ def _inflateInit2(stream, wbits): _deflateSetDictionary = zlib_external('deflateSetDictionary', [z_stream_p, Bytefp, uInt], rffi.INT) _inflateSetDictionary = zlib_external('inflateSetDictionary', [z_stream_p, Bytefp, uInt], rffi.INT) +_zlibVersion = zlib_external('zlibVersion', [], rffi.CCHARP) # ____________________________________________________________ @@ -208,6 +209,10 @@ def adler32(string, start=ADLER32_DEFAULT_START): checksum = _adler32(start, rffi.cast(Bytefp, bytes), len(string)) return checksum +def zlibVersion(): + """Return the runtime version of zlib library""" + return rffi.charp2str(_zlibVersion()) + # ____________________________________________________________ class RZlibError(Exception): diff --git a/rpython/rtyper/module/test/test_ll_termios.py b/rpython/rlib/test/test_rtermios.py index 08c7e5b69b..5b63459afd 100644 --- a/rpython/rtyper/module/test/test_ll_termios.py +++ b/rpython/rlib/test/test_rtermios.py @@ -77,13 +77,12 @@ class TestLLTermios(object): def test_tcrest(self): from rpython.translator.c.test.test_genc import compile - from rpython.rtyper.module import ll_termios - import termios, time + from rpython.rlib import rtermios def runs_tcall(): - termios.tcsendbreak(2, 0) - termios.tcdrain(2) - termios.tcflush(2, termios.TCIOFLUSH) - termios.tcflow(2, termios.TCOON) + rtermios.tcsendbreak(2, 0) + rtermios.tcdrain(2) + rtermios.tcflush(2, rtermios.TCIOFLUSH) + rtermios.tcflow(2, rtermios.TCOON) print "ok" fn = compile(runs_tcall, [], backendopt=False) diff --git a/rpython/rlib/test/test_rzlib.py b/rpython/rlib/test/test_rzlib.py index 6556027fab..faaaffe461 100644 --- a/rpython/rlib/test/test_rzlib.py +++ b/rpython/rlib/test/test_rzlib.py @@ -269,3 +269,7 @@ def test_cornercases(): assert unused > 0 buf = buf[-unused:] rzlib.deflateEnd(stream) + +def test_zlibVersion(): + runtime_version = rzlib.zlibVersion() + assert runtime_version[0] == rzlib.ZLIB_VERSION[0] diff --git a/rpython/rtyper/extfuncregistry.py b/rpython/rtyper/extfuncregistry.py index b73e4520fe..0d962075d5 100644 --- a/rpython/rtyper/extfuncregistry.py +++ b/rpython/rtyper/extfuncregistry.py @@ -10,12 +10,6 @@ from rpython.rtyper.lltypesystem.module import ll_math from rpython.rtyper.module import ll_os from rpython.rtyper.module import ll_time from rpython.rlib import rfloat -try: - import termios -except ImportError: - pass -else: - from rpython.rtyper.module import ll_termios # the following functions all take one float, return one float # and are part of math.h diff --git a/rpython/rtyper/llinterp.py b/rpython/rtyper/llinterp.py index f3f5a70074..01e54ad5b6 100644 --- a/rpython/rtyper/llinterp.py +++ b/rpython/rtyper/llinterp.py @@ -911,6 +911,9 @@ class LLFrame(object): def op_gc_typeids_z(self): raise NotImplementedError("gc_typeids_z") + def op_gc_typeids_list(self): + raise NotImplementedError("gc_typeids_list") + def op_gc_gcflag_extra(self, subopnum, *args): return self.heap.gcflag_extra(subopnum, *args) diff --git a/rpython/rtyper/lltypesystem/lloperation.py b/rpython/rtyper/lltypesystem/lloperation.py index cc19712416..94ad6dac19 100644 --- a/rpython/rtyper/lltypesystem/lloperation.py +++ b/rpython/rtyper/lltypesystem/lloperation.py @@ -493,6 +493,7 @@ LL_OPERATIONS = { 'gc_is_rpy_instance' : LLOp(), 'gc_dump_rpy_heap' : LLOp(), 'gc_typeids_z' : LLOp(), + 'gc_typeids_list' : LLOp(), 'gc_gcflag_extra' : LLOp(), 'gc_add_memory_pressure': LLOp(), diff --git a/rpython/rtyper/lltypesystem/rffi.py b/rpython/rtyper/lltypesystem/rffi.py index 49d1d28978..adc48d5670 100644 --- a/rpython/rtyper/lltypesystem/rffi.py +++ b/rpython/rtyper/lltypesystem/rffi.py @@ -471,11 +471,21 @@ try: except CompilationError: pass -_TYPES_ARE_UNSIGNED = set(['size_t', 'uintptr_t']) # plus "unsigned *" if os.name != 'nt': TYPES.append('mode_t') TYPES.append('pid_t') TYPES.append('ssize_t') + # the types below are rare enough and not available on Windows + TYPES.extend(['ptrdiff_t', + 'int_least8_t', 'uint_least8_t', + 'int_least16_t', 'uint_least16_t', + 'int_least32_t', 'uint_least32_t', + 'int_least64_t', 'uint_least64_t', + 'int_fast8_t', 'uint_fast8_t', + 'int_fast16_t', 'uint_fast16_t', + 'int_fast32_t', 'uint_fast32_t', + 'int_fast64_t', 'uint_fast64_t', + 'intmax_t', 'uintmax_t']) else: MODE_T = lltype.Signed PID_T = lltype.Signed @@ -489,8 +499,10 @@ def populate_inttypes(): if name.startswith('unsigned'): name = 'u' + name[9:] signed = False + elif name == 'size_t' or name.startswith('uint'): + signed = False else: - signed = (name not in _TYPES_ARE_UNSIGNED) + signed = True name = name.replace(' ', '') names.append(name) populatelist.append((name.upper(), c_name, signed)) diff --git a/rpython/rtyper/module/ll_termios.py b/rpython/rtyper/module/ll_termios.py deleted file mode 100644 index 1ed3b4513a..0000000000 --- a/rpython/rtyper/module/ll_termios.py +++ /dev/null @@ -1,135 +0,0 @@ - -""" -The low-level implementation of termios module -note that this module should only be imported when -termios module is there -""" - -import termios -from rpython.rtyper.lltypesystem import rffi -from rpython.rtyper.lltypesystem import lltype -from rpython.rtyper.extfunc import lazy_register, register_external -from rpython.rlib.rarithmetic import intmask -from rpython.rtyper.extregistry import ExtRegistryEntry -from rpython.annotator import model as annmodel -from rpython.rtyper import rclass -from rpython.rlib import rtermios, rposix -from rpython.rtyper.tool import rffi_platform -from rpython.translator.tool.cbuild import ExternalCompilationInfo - -eci = ExternalCompilationInfo( - includes = ['termios.h', 'unistd.h'] -) - -class CConfig: - _compilation_info_ = eci - NCCS = rffi_platform.DefinedConstantInteger('NCCS') - _HAVE_STRUCT_TERMIOS_C_ISPEED = rffi_platform.Defined( - '_HAVE_STRUCT_TERMIOS_C_ISPEED') - _HAVE_STRUCT_TERMIOS_C_OSPEED = rffi_platform.Defined( - '_HAVE_STRUCT_TERMIOS_C_OSPEED') - -c_config = rffi_platform.configure(CConfig) -NCCS = c_config['NCCS'] - -TCFLAG_T = rffi.UINT -CC_T = rffi.UCHAR -SPEED_T = rffi.UINT -INT = rffi.INT - -_add = [] -if c_config['_HAVE_STRUCT_TERMIOS_C_ISPEED']: - _add.append(('c_ispeed', SPEED_T)) -if c_config['_HAVE_STRUCT_TERMIOS_C_OSPEED']: - _add.append(('c_ospeed', SPEED_T)) -TERMIOSP = rffi.CStructPtr('termios', ('c_iflag', TCFLAG_T), ('c_oflag', TCFLAG_T), - ('c_cflag', TCFLAG_T), ('c_lflag', TCFLAG_T), - ('c_line', CC_T), - ('c_cc', lltype.FixedSizeArray(CC_T, NCCS)), *_add) - -def c_external(name, args, result): - return rffi.llexternal(name, args, result, compilation_info=eci) - -c_tcsetattr = c_external('tcsetattr', [INT, INT, TERMIOSP], INT) -c_cfgetispeed = c_external('cfgetispeed', [TERMIOSP], SPEED_T) -c_cfgetospeed = c_external('cfgetospeed', [TERMIOSP], SPEED_T) -c_cfsetispeed = c_external('cfsetispeed', [TERMIOSP, SPEED_T], INT) -c_cfsetospeed = c_external('cfsetospeed', [TERMIOSP, SPEED_T], INT) -c_tcsendbreak = c_external('tcsendbreak', [INT, INT], INT) -c_tcdrain = c_external('tcdrain', [INT], INT) -c_tcflush = c_external('tcflush', [INT, INT], INT) -c_tcflow = c_external('tcflow', [INT, INT], INT) - -c_tcgetattr = c_external('tcgetattr', [INT, TERMIOSP], INT) - -def tcgetattr_llimpl(fd): - c_struct = lltype.malloc(TERMIOSP.TO, flavor='raw') - - try: - if c_tcgetattr(fd, c_struct) < 0: - raise OSError(rposix.get_errno(), 'tcgetattr failed') - cc = [chr(c_struct.c_c_cc[i]) for i in range(NCCS)] - ispeed = c_cfgetispeed(c_struct) - ospeed = c_cfgetospeed(c_struct) - result = (intmask(c_struct.c_c_iflag), intmask(c_struct.c_c_oflag), - intmask(c_struct.c_c_cflag), intmask(c_struct.c_c_lflag), - intmask(ispeed), intmask(ospeed), cc) - return result - finally: - lltype.free(c_struct, flavor='raw') - -register_external(rtermios.tcgetattr, [int], (int, int, int, int, int, int, [str]), - llimpl=tcgetattr_llimpl, export_name='termios.tcgetattr') - -def tcsetattr_llimpl(fd, when, attributes): - c_struct = lltype.malloc(TERMIOSP.TO, flavor='raw') - try: - c_struct.c_c_iflag = r_uint(attributes[0]) - c_struct.c_c_oflag = r_uint(attributes[1]) - c_struct.c_c_cflag = r_uint(attributes[2]) - c_struct.c_c_lflag = r_uint(attributes[3]) - ispeed = r_uint(attributes[4]) - ospeed = r_uint(attributes[5]) - cc = attributes[6] - for i in range(NCCS): - c_struct.c_c_cc[i] = rffi.r_uchar(ord(cc[i][0])) - if c_cfsetispeed(c_struct, ispeed) < 0: - raise OSError(rposix.get_errno(), 'tcsetattr failed') - if c_cfsetospeed(c_struct, ospeed) < 0: - raise OSError(rposix.get_errno(), 'tcsetattr failed') - if c_tcsetattr(fd, when, c_struct) < 0: - raise OSError(rposix.get_errno(), 'tcsetattr failed') - finally: - lltype.free(c_struct, flavor='raw') - -r_uint = rffi.r_uint -register_external(rtermios.tcsetattr, [int, int, (int, int, int, - int, int, int, [str])], llimpl=tcsetattr_llimpl, - export_name='termios.tcsetattr') - -# a bit C-c C-v code follows... - -def tcsendbreak_llimpl(fd, duration): - if c_tcsendbreak(fd, duration): - raise OSError(rposix.get_errno(), 'tcsendbreak failed') -register_external(termios.tcsendbreak, [int, int], - llimpl=tcsendbreak_llimpl, - export_name='termios.tcsendbreak') - -def tcdrain_llimpl(fd): - if c_tcdrain(fd) < 0: - raise OSError(rposix.get_errno(), 'tcdrain failed') -register_external(termios.tcdrain, [int], llimpl=tcdrain_llimpl, - export_name='termios.tcdrain') - -def tcflush_llimpl(fd, queue_selector): - if c_tcflush(fd, queue_selector) < 0: - raise OSError(rposix.get_errno(), 'tcflush failed') -register_external(termios.tcflush, [int, int], llimpl=tcflush_llimpl, - export_name='termios.tcflush') - -def tcflow_llimpl(fd, action): - if c_tcflow(fd, action) < 0: - raise OSError(rposix.get_errno(), 'tcflow failed') -register_external(termios.tcflow, [int, int], llimpl=tcflow_llimpl, - export_name='termios.tcflow') diff --git a/rpython/rtyper/module/ll_time.py b/rpython/rtyper/module/ll_time.py index 9b7634509e..638c7e158c 100644 --- a/rpython/rtyper/module/ll_time.py +++ b/rpython/rtyper/module/ll_time.py @@ -47,6 +47,8 @@ class CConfig: if sys.platform.startswith('freebsd') or sys.platform.startswith('netbsd'): libraries = ['compat'] +elif sys.platform == 'linux2': + libraries = ['rt'] else: libraries = [] @@ -58,7 +60,15 @@ class CConfigForFTime: TIMEB = platform.Struct(STRUCT_TIMEB, [('time', rffi.INT), ('millitm', rffi.INT)]) -constant_names = ['RUSAGE_SELF', 'EINTR'] +class CConfigForClockGetTime: + _compilation_info_ = ExternalCompilationInfo( + includes=['time.h'], + libraries=libraries + ) + TIMESPEC = platform.Struct('struct timespec', [('tv_sec', rffi.LONG), + ('tv_nsec', rffi.LONG)]) + +constant_names = ['RUSAGE_SELF', 'EINTR', 'CLOCK_PROCESS_CPUTIME_ID'] for const in constant_names: setattr(CConfig, const, platform.DefinedConstantInteger(const)) defs_names = ['GETTIMEOFDAY_NO_TZ'] @@ -162,6 +172,21 @@ class RegisterTime(BaseLazyRegistering): diff = a[0] - state.counter_start lltype.free(a, flavor='raw') return float(diff) / state.divisor + elif self.CLOCK_PROCESS_CPUTIME_ID is not None: + # Linux and other POSIX systems with clock_gettime() + self.configure(CConfigForClockGetTime) + TIMESPEC = self.TIMESPEC + CLOCK_PROCESS_CPUTIME_ID = self.CLOCK_PROCESS_CPUTIME_ID + c_clock_gettime = self.llexternal('clock_gettime', + [lltype.Signed, lltype.Ptr(TIMESPEC)], + rffi.INT, releasegil=False) + def time_clock_llimpl(): + a = lltype.malloc(TIMESPEC, flavor='raw') + c_clock_gettime(CLOCK_PROCESS_CPUTIME_ID, a) + result = (float(rffi.getintfield(a, 'c_tv_sec')) + + float(rffi.getintfield(a, 'c_tv_nsec')) * 0.000000001) + lltype.free(a, flavor='raw') + return result else: RUSAGE = self.RUSAGE RUSAGE_SELF = self.RUSAGE_SELF or 0 diff --git a/rpython/rtyper/tool/rfficache.py b/rpython/rtyper/tool/rfficache.py index 7f2f4d2e1b..559d2ff683 100644 --- a/rpython/rtyper/tool/rfficache.py +++ b/rpython/rtyper/tool/rfficache.py @@ -14,7 +14,7 @@ def ask_gcc(question, add_source="", ignore_errors=False): from rpython.translator.platform import platform includes = ['stdlib.h', 'stdio.h', 'sys/types.h'] if platform.name != 'msvc': - includes += ['inttypes.h'] + includes += ['inttypes.h', 'stddef.h'] include_string = "\n".join(["#include <%s>" % i for i in includes]) c_source = py.code.Source(''' // includes diff --git a/rpython/tool/algo/graphlib.py b/rpython/tool/algo/graphlib.py index 652d9ebbb0..1e40d4a587 100644 --- a/rpython/tool/algo/graphlib.py +++ b/rpython/tool/algo/graphlib.py @@ -182,6 +182,8 @@ def break_cycles(vertices, edges): """Enumerates a reasonably minimal set of edges that must be removed to make the graph acyclic.""" + import py; py.test.skip("break_cycles() is not used any more") + # the approach is as follows: starting from each root, find some set # of cycles using a simple depth-first search. Then break the # edge that is part of the most cycles. Repeat. diff --git a/rpython/tool/jitlogparser/parser.py b/rpython/tool/jitlogparser/parser.py index fc001d23ff..ee8f64155b 100644 --- a/rpython/tool/jitlogparser/parser.py +++ b/rpython/tool/jitlogparser/parser.py @@ -27,7 +27,7 @@ class Op(object): asm = None failargs = () - def __init__(self, name, args, res, descr): + def __init__(self, name, args, res, descr, failargs=None): self.name = name self.args = args self.res = res @@ -35,6 +35,7 @@ class Op(object): self._is_guard = name.startswith('guard_') if self._is_guard: self.guard_no = int(self.descr[len('<Guard0x'):-1], 16) + self.failargs = failargs def as_json(self): d = { @@ -155,8 +156,8 @@ class SimpleParser(OpParser): def box_for_var(self, res): return res - def create_op(self, opnum, args, res, descr): - return self.Op(intern(opname[opnum].lower()), args, res, descr) + def create_op(self, opnum, args, res, descr, fail_args): + return self.Op(intern(opname[opnum].lower()), args, res, descr, fail_args) diff --git a/rpython/translator/c/gcc/trackgcroot.py b/rpython/translator/c/gcc/trackgcroot.py index 7bf932338b..75ea9625f0 100755 --- a/rpython/translator/c/gcc/trackgcroot.py +++ b/rpython/translator/c/gcc/trackgcroot.py @@ -588,13 +588,6 @@ class FunctionGcRootTracker(object): else: return [] - # The various cmov* operations - for name in ''' - e ne g ge l le a ae b be nb p np s ns o no - '''.split(): - locals()['visit_cmov' + name] = binary_insn - locals()['visit_cmov' + name + 'l'] = binary_insn - def _visit_and(self, line): match = self.r_binaryinsn.match(line) target = match.group("target") @@ -828,23 +821,18 @@ class FunctionGcRootTracker(object): return prefix + [InsnCondJump(label)] + postfix visit_jmpl = visit_jmp - visit_jg = conditional_jump - visit_jge = conditional_jump - visit_jl = conditional_jump - visit_jle = conditional_jump - visit_ja = conditional_jump - visit_jae = conditional_jump - visit_jb = conditional_jump - visit_jbe = conditional_jump - visit_jp = conditional_jump - visit_jnb = conditional_jump - visit_jnp = conditional_jump - visit_js = conditional_jump - visit_jns = conditional_jump - visit_jo = conditional_jump - visit_jno = conditional_jump - visit_jc = conditional_jump - visit_jnc = conditional_jump + + # The various conditional jumps and cmov* operations + for name in ''' + e g ge l le a ae b be p s o c + '''.split(): + # NB. visit_je() and visit_jne() are overridden below + locals()['visit_j' + name] = conditional_jump + locals()['visit_jn' + name] = conditional_jump + locals()['visit_cmov' + name] = binary_insn + locals()['visit_cmov' + name + 'l'] = binary_insn + locals()['visit_cmovn' + name] = binary_insn + locals()['visit_cmovn' + name + 'l'] = binary_insn def visit_je(self, line): return self.conditional_jump(line, je=True) @@ -1526,6 +1514,9 @@ class ElfAssemblerParser(AssemblerParser): yield True, functionlines in_function = False functionlines = [] + if in_function and ".get_pc_thunk.bx" in functionlines[0]: + in_function = False # xxx? ignore this rare unclosed stub at + # the end of the file assert not in_function, ( "missed the end of the previous function") yield False, functionlines diff --git a/rpython/translator/c/genc.py b/rpython/translator/c/genc.py index 4d258c9866..8ad65ea9de 100644 --- a/rpython/translator/c/genc.py +++ b/rpython/translator/c/genc.py @@ -400,15 +400,15 @@ class CStandaloneBuilder(CBuilder): mk.definition('PROFOPT', profopt) rules = [ - ('clean', '', 'rm -f $(OBJECTS) $(TARGET) $(GCMAPFILES) $(ASMFILES) *.gc?? ../module_cache/*.gc??'), - ('clean_noprof', '', 'rm -f $(OBJECTS) $(TARGET) $(GCMAPFILES) $(ASMFILES)'), + ('clean', '', 'rm -f $(OBJECTS) $(DEFAULT_TARGET) $(TARGET) $(GCMAPFILES) $(ASMFILES) *.gc?? ../module_cache/*.gc??'), + ('clean_noprof', '', 'rm -f $(OBJECTS) $(DEFAULT_TARGET) $(TARGET) $(GCMAPFILES) $(ASMFILES)'), ('debug', '', '$(MAKE) CFLAGS="$(DEBUGFLAGS) -DRPY_ASSERT" debug_target'), ('debug_exc', '', '$(MAKE) CFLAGS="$(DEBUGFLAGS) -DRPY_ASSERT -DDO_LOG_EXC" debug_target'), ('debug_mem', '', '$(MAKE) CFLAGS="$(DEBUGFLAGS) -DRPY_ASSERT -DPYPY_USE_TRIVIAL_MALLOC" debug_target'), - ('llsafer', '', '$(MAKE) CFLAGS="-O2 -DRPY_LL_ASSERT" $(TARGET)'), + ('llsafer', '', '$(MAKE) CFLAGS="-O2 -DRPY_LL_ASSERT" $(DEFAULT_TARGET)'), ('lldebug', '', '$(MAKE) CFLAGS="$(DEBUGFLAGS) -DRPY_ASSERT -DRPY_LL_ASSERT" debug_target'), ('lldebug0','', '$(MAKE) CFLAGS="$(DEBUGFLAGS) -O0 -DRPY_ASSERT -DRPY_LL_ASSERT" debug_target'), - ('profile', '', '$(MAKE) CFLAGS="-g -O1 -pg $(CFLAGS) -fno-omit-frame-pointer" LDFLAGS="-pg $(LDFLAGS)" $(TARGET)'), + ('profile', '', '$(MAKE) CFLAGS="-g -O1 -pg $(CFLAGS) -fno-omit-frame-pointer" LDFLAGS="-pg $(LDFLAGS)" $(DEFAULT_TARGET)'), ] if self.has_profopt(): rules.append( @@ -449,7 +449,7 @@ class CStandaloneBuilder(CBuilder): '-o $*.s -S $< $(INCLUDEDIRS)', '$(PYTHON) $(RPYDIR)/translator/c/gcc/trackgcroot.py ' '-t $*.s > $*.gctmp', - '$(CC) -o $*.o -c $*.lbl.s', + '$(CC) $(CFLAGS) -o $*.o -c $*.lbl.s', 'mv $*.gctmp $*.gcmap', 'rm $*.s $*.lbl.s']) @@ -471,7 +471,7 @@ class CStandaloneBuilder(CBuilder): if self.translator.platform.name == 'msvc': mk.rule('debug_target', 'debugmode_$(DEFAULT_TARGET)', 'rem') else: - mk.rule('debug_target', '$(TARGET)', '#') + mk.rule('debug_target', '$(DEFAULT_TARGET)', '#') mk.write() #self.translator.platform, # , diff --git a/rpython/translator/c/test/test_newgc.py b/rpython/translator/c/test/test_newgc.py index 234000a80a..3eaf4352bc 100644 --- a/rpython/translator/c/test/test_newgc.py +++ b/rpython/translator/c/test/test_newgc.py @@ -1147,6 +1147,10 @@ class UsingFrameworkTest(object): fd = os.open(filename, open_flags, 0666) os.write(fd, s) os.close(fd) + # + a = rgc.get_typeids_list() + assert len(a) > 1 + assert 0 < rffi.cast(lltype.Signed, a[1]) < 10000 return 0 return fn diff --git a/rpython/translator/c/test/test_standalone.py b/rpython/translator/c/test/test_standalone.py index 15ee4030ae..0320d1c980 100644 --- a/rpython/translator/c/test/test_standalone.py +++ b/rpython/translator/c/test/test_standalone.py @@ -1386,6 +1386,8 @@ class TestShared(StandaloneTests): ext_suffix = '.so' if cbuilder.eci.platform.name == 'msvc': ext_suffix = '.dll' + elif cbuilder.eci.platform.name.startswith('darwin'): + ext_suffix = '.dylib' libname = cbuilder.executable_name.join('..', 'lib' + cbuilder.modulename + ext_suffix) lib = ctypes.CDLL(str(libname)) diff --git a/rpython/translator/platform/darwin.py b/rpython/translator/platform/darwin.py index b0ffece637..8c4baad6ab 100644 --- a/rpython/translator/platform/darwin.py +++ b/rpython/translator/platform/darwin.py @@ -10,10 +10,11 @@ class Darwin(posix.BasePosix): so_ext = 'dylib' DEFAULT_CC = 'clang' + rpath_flags = ['-Wl,-rpath', '-Wl,@executable_path/'] def _args_for_shared(self, args): return (list(self.shared_only) - + ['-dynamiclib', '-undefined', 'dynamic_lookup'] + + ['-dynamiclib', '-install_name', '@rpath/$(TARGET)', '-undefined', 'dynamic_lookup'] + args) def _include_dirs_for_libffi(self): diff --git a/rpython/translator/platform/freebsd.py b/rpython/translator/platform/freebsd.py index 20c952715c..95c5303a73 100644 --- a/rpython/translator/platform/freebsd.py +++ b/rpython/translator/platform/freebsd.py @@ -6,9 +6,12 @@ from rpython.translator.platform.bsd import BSD class Freebsd(BSD): name = "freebsd" - link_flags = ['-pthread'] + os.environ.get('LDFLAGS', '').split() - cflags = ['-O3', '-pthread', '-fomit-frame-pointer' - ] + os.environ.get('CFLAGS', '').split() + link_flags = tuple( + ['-pthread'] + + os.environ.get('LDFLAGS', '').split()) + cflags = tuple( + ['-O3', '-pthread', '-fomit-frame-pointer'] + + os.environ.get('CFLAGS', '').split()) class Freebsd_64(Freebsd): shared_only = ('-fPIC',) diff --git a/rpython/translator/platform/posix.py b/rpython/translator/platform/posix.py index 47534bc8b2..ddc4ee98ab 100644 --- a/rpython/translator/platform/posix.py +++ b/rpython/translator/platform/posix.py @@ -14,6 +14,7 @@ class BasePosix(Platform): relevant_environ = ('CPATH', 'LIBRARY_PATH', 'C_INCLUDE_PATH') DEFAULT_CC = 'gcc' + rpath_flags = ['-Wl,-rpath=\'$$ORIGIN/\''] def __init__(self, cc=None): self.cc = cc or os.environ.get('CC', self.DEFAULT_CC) @@ -158,6 +159,7 @@ class BasePosix(Platform): ('CC', self.cc), ('CC_LINK', eci.use_cpp_linker and 'g++' or '$(CC)'), ('LINKFILES', eci.link_files), + ('RPATH_FLAGS', self.rpath_flags), ] for args in definitions: m.definition(*args) @@ -181,7 +183,7 @@ class BasePosix(Platform): 'int main(int argc, char* argv[]) ' '{ return $(PYPY_MAIN_FUNCTION)(argc, argv); }" > $@') m.rule('$(DEFAULT_TARGET)', ['$(TARGET)', 'main.o'], - '$(CC_LINK) $(LDFLAGS_LINK) main.o -L. -l$(SHARED_IMPORT_LIB) -o $@ -Wl,-rpath=\'$$ORIGIN/\'') + '$(CC_LINK) $(LDFLAGS_LINK) main.o -L. -l$(SHARED_IMPORT_LIB) -o $@ $(RPATH_FLAGS)') return m diff --git a/rpython/translator/platform/windows.py b/rpython/translator/platform/windows.py index fc13c2feea..8a00d1fed4 100644 --- a/rpython/translator/platform/windows.py +++ b/rpython/translator/platform/windows.py @@ -293,7 +293,7 @@ class MsvcPlatform(Platform): m.comment('automatically generated makefile') definitions = [ - ('RPYDIR', rpydir), + ('RPYDIR', '"%s"' % rpydir), ('TARGET', target_name), ('DEFAULT_TARGET', exe_name.basename), ('SOURCES', rel_cfiles), |