diff options
| author |   Hakan Ardo <hakan@debian.org> | 2010-11-17 19:41:25 +0000 |
|---|---|---|
| committer | Hakan Ardo <hakan@debian.org> | 2010-11-17 19:41:25 +0000 |
| commit | baf678c574081d15f0e409eb8d004c76e87b95c2 (patch) | |
| tree | 1b7f8e03b86f24a6c8a44ece82fcdb6e5602fce4 /pypy | |
| parent | Better name (diff) | |
| download | pypy-baf678c574081d15f0e409eb8d004c76e87b95c2.tar.gz pypy-baf678c574081d15f0e409eb8d004c76e87b95c2.tar.bz2 pypy-baf678c574081d15f0e409eb8d004c76e87b95c2.zip | |
svn merge -r78744:HEAD svn+ssh://hakanardo@codespeak.net/svn/pypy/trunk
Diffstat (limited to 'pypy')
140 files changed, 2337 insertions, 838 deletions
diff --git a/pypy/annotation/bookkeeper.py b/pypy/annotation/bookkeeper.py index 95037dc86b..ee596315eb 100644 --- a/pypy/annotation/bookkeeper.py +++ b/pypy/annotation/bookkeeper.py @@ -87,13 +87,6 @@ class Stats(object): else: return obj.knowntype.__name__ - def consider_tuple_iter(self, tup): - ctxt = "[%s]" % sys._getframe(4).f_code.co_name - if tup.is_constant(): - return ctxt, tup.const - else: - return ctxt, tuple([self.typerepr(x) for x in tup.items]) - def consider_tuple_random_getitem(self, tup): return tuple([self.typerepr(x) for x in tup.items]) diff --git a/pypy/annotation/builtin.py b/pypy/annotation/builtin.py index 97329fa8cf..acc7453a8b 100644 --- a/pypy/annotation/builtin.py +++ b/pypy/annotation/builtin.py @@ -453,6 +453,9 @@ def free(s_p, s_flavor, s_track_allocation=None): #p = lltype.malloc(T, flavor=s_flavor.const) #lltype.free(p, flavor=s_flavor.const) +def render_immortal(s_p, s_track_allocation=None): + assert s_track_allocation is None or s_track_allocation.is_constant() + def typeOf(s_val): lltype = annotation_to_lltype(s_val, info="in typeOf(): ") return immutablevalue(lltype) @@ -520,6 +523,7 @@ def constPtr(T): BUILTIN_ANALYZERS[lltype.malloc] = malloc BUILTIN_ANALYZERS[lltype.free] = free +BUILTIN_ANALYZERS[lltype.render_immortal] = render_immortal BUILTIN_ANALYZERS[lltype.typeOf] = typeOf BUILTIN_ANALYZERS[lltype.cast_primitive] = cast_primitive BUILTIN_ANALYZERS[lltype.nullptr] = nullptr diff --git a/pypy/config/pypyoption.py b/pypy/config/pypyoption.py index 0994f3e8e7..3bf5434d15 100644 --- a/pypy/config/pypyoption.py +++ b/pypy/config/pypyoption.py @@ -34,6 +34,11 @@ working_modules.update(dict.fromkeys( "_bisect"] )) +translation_modules = default_modules.copy() +translation_modules.update(dict.fromkeys( + ["fcntl", "rctime", "select", "signal", "_rawffi", "zlib", + "struct", "md5", "cStringIO", "array"])) + working_oo_modules = default_modules.copy() working_oo_modules.update(dict.fromkeys( ["md5", "sha", "cStringIO", "itertools"] @@ -149,6 +154,12 @@ pypy_optiondescription = OptionDescription("objspace", "Object Space Options", [ cmdline="--allworkingmodules", negation=True), + BoolOption("translationmodules", + "use only those modules that are needed to run translate.py on pypy", + default=False, + cmdline="--translationmodules", + suggests=[("objspace.allworkingmodules", False)]), + BoolOption("geninterp", "specify whether geninterp should be used", cmdline=None, default=True), @@ -164,6 +175,11 @@ pypy_optiondescription = OptionDescription("objspace", "Object Space Options", [ default=False, requires=[("objspace.usepycfiles", True)]), + StrOption("soabi", + "Tag to differentiate extension modules built for different Python interpreters", + cmdline="--soabi", + default=None), + BoolOption("honor__builtins__", "Honor the __builtins__ key of a module dictionary", default=False), @@ -369,6 +385,11 @@ def enable_allworkingmodules(config): modules = [name for name in modules if name not in essential_modules] config.objspace.usemodules.suggest(**dict.fromkeys(modules, True)) +def enable_translationmodules(config): + modules = translation_modules + modules = [name for name in modules if name not in essential_modules] + config.objspace.usemodules.suggest(**dict.fromkeys(modules, True)) + if __name__ == '__main__': config = get_pypy_config() diff --git a/pypy/conftest.py b/pypy/conftest.py index 09c38c6d69..7d0146f586 100644 --- a/pypy/conftest.py +++ b/pypy/conftest.py @@ -327,6 +327,31 @@ class AppError(Exception): class PyPyTestFunction(py.test.collect.Function): # All PyPy test items catch and display OperationErrors specially. # + def runtest(self): + self.runtest_open() + try: + self.runtest_perform() + finally: + self.runtest_close() + self.runtest_finish() + + def runtest_open(self): + leakfinder.start_tracking_allocations() + + def runtest_perform(self): + super(PyPyTestFunction, self).runtest() + + def runtest_close(self): + if leakfinder.TRACK_ALLOCATIONS: + self._pypytest_leaks = leakfinder.stop_tracking_allocations(False) + else: # stop_tracking_allocations() already called + self._pypytest_leaks = None + + def runtest_finish(self): + # check for leaks, but only if the test passed so far + if self._pypytest_leaks: + raise leakfinder.MallocMismatch(self._pypytest_leaks) + def execute_appex(self, space, target, *args): try: target(*args) @@ -353,16 +378,9 @@ class IntTestFunction(PyPyTestFunction): def _keywords(self): return super(IntTestFunction, self)._keywords() + ['interplevel'] - def runtest(self): + def runtest_perform(self): try: - leakfinder.start_tracking_allocations() - try: - super(IntTestFunction, self).runtest() - finally: - if leakfinder.TRACK_ALLOCATIONS: - leaks = leakfinder.stop_tracking_allocations(False) - else: - leaks = None # stop_tracking_allocations() already called + super(IntTestFunction, self).runtest_perform() except OperationError, e: check_keyboard_interrupt(e) raise @@ -375,14 +393,15 @@ class IntTestFunction(PyPyTestFunction): py.test.skip('%s: %s' % (e.__class__.__name__, e)) cls = cls.__bases__[0] raise + + def runtest_finish(self): if 'pygame' in sys.modules: global _pygame_imported if not _pygame_imported: _pygame_imported = True assert option.view, ("should not invoke Pygame " "if conftest.option.view is False") - if leaks: # check for leaks, but only if the test passed so far - raise leakfinder.MallocMismatch(leaks) + super(IntTestFunction, self).runtest_finish() class AppTestFunction(PyPyTestFunction): def _prunetraceback(self, traceback): @@ -395,7 +414,7 @@ class AppTestFunction(PyPyTestFunction): def _keywords(self): return ['applevel'] + super(AppTestFunction, self)._keywords() - def runtest(self): + def runtest_perform(self): target = self.obj if option.runappdirect: return target() @@ -427,7 +446,7 @@ class AppTestMethod(AppTestFunction): space.setattr(w_instance, space.wrap(name[2:]), getattr(instance, name)) - def runtest(self): + def runtest_perform(self): target = self.obj if option.runappdirect: return target() diff --git a/pypy/doc/config/objspace.soabi.txt b/pypy/doc/config/objspace.soabi.txt new file mode 100644 index 0000000000..ec55f28449 --- /dev/null +++ b/pypy/doc/config/objspace.soabi.txt @@ -0,0 +1,14 @@ +This option controls the tag included into extension module file names. The +default is something like `pypy-14`, which means that `import foo` will look for +a file named `foo.pypy-14.so` (or `foo.pypy-14.pyd` on Windows). + +This is an implementation of PEP3149_, with two differences: + + * the filename without tag `foo.so` is not considered. + * the feature is also available on Windows. + +When set to the empty string (with `--soabi=`), the interpreter will only look +for a file named `foo.so`, and will crash if this file was compiled for another +Python interpreter. + +.. _PEP3149: http://www.python.org/dev/peps/pep-3149/ diff --git a/pypy/doc/config/objspace.translationmodules.txt b/pypy/doc/config/objspace.translationmodules.txt new file mode 100644 index 0000000000..532e771524 --- /dev/null +++ b/pypy/doc/config/objspace.translationmodules.txt @@ -0,0 +1 @@ +This option enables all modules which are needed to translate PyPy using PyPy. diff --git a/pypy/doc/release-1.4.0.txt b/pypy/doc/release-1.4.0.txt new file mode 100644 index 0000000000..68fc83f7d4 --- /dev/null +++ b/pypy/doc/release-1.4.0.txt @@ -0,0 +1,28 @@ +=============================== +PyPy 1.4: Ouroboros in practice +=============================== + +Hello. + +We're pleased to announce the 1.4 release of PyPy. This is a major breakthrough +in our long journey, as PyPy 1.4 is the first PyPy release that can translate +itself faster than CPython. Starting today, we plan to start using PyPy for our +own development. + +Among other features, this release includes numerous performance improvements +(which made fast self-hosting possible), a 64-bit JIT backend, as well as serious +stabilization. As of now, we can consider the 32-bit version of PyPy stable enough +to run in production. + +More highlights +=============== + +* Virtualenv support: now PyPy is fully compatible with virtualenv_: note that + to use it, you need a recent version of virtualenv (>= 1.5). + +* Faster (and JITted) regular expressions - huge boost in speeding up + sre module. + +* Faster (and JITted) calls to functions like map(). + +.. _virtualenv: http://pypi.python.org/pypi/virtualenv diff --git a/pypy/interpreter/argument.py b/pypy/interpreter/argument.py index 4aa9ecefd8..09d3aefa8b 100644 --- a/pypy/interpreter/argument.py +++ b/pypy/interpreter/argument.py @@ -64,7 +64,7 @@ class Signature(object): return not self == other - # make it look tuply for the annotator + # make it look tuply for its use in the annotator def __len__(self): return 3 @@ -103,10 +103,11 @@ class Arguments(object): make_sure_not_resized(self.keywords_w) make_sure_not_resized(self.arguments_w) - if ((w_stararg is not None and w_stararg) or - (w_starstararg is not None and w_starstararg)): - self._combine_wrapped(w_stararg, w_starstararg) - # if we have a call where * or ** args are used at the callsite + if w_stararg is not None and space.is_true(w_stararg): + self._combine_starargs_wrapped(w_stararg) + if w_starstararg is not None and space.is_true(w_starstararg): + self._combine_starstarargs_wrapped(w_starstararg) + # if we have a call where **args are used at the callsite # we shouldn't let the JIT see the argument matching self._dont_jit = True else: @@ -142,42 +143,48 @@ class Arguments(object): def _combine_wrapped(self, w_stararg, w_starstararg): "unpack the *arg and **kwd into arguments_w and keywords_w" - # unpack the * arguments if w_stararg is not None: - self.arguments_w = (self.arguments_w + - self.space.fixedview(w_stararg)) - # unpack the ** arguments + self._combine_starargs_wrapped(w_stararg) if w_starstararg is not None: - space = self.space - if not space.is_true(space.isinstance(w_starstararg, space.w_dict)): + self._combine_starstarargs_wrapped(w_starstararg) + + def _combine_starargs_wrapped(self, w_stararg): + # unpack the * arguments + self.arguments_w = (self.arguments_w + + self.space.fixedview(w_stararg)) + + def _combine_starstarargs_wrapped(self, w_starstararg): + # unpack the ** arguments + space = self.space + if not space.is_true(space.isinstance(w_starstararg, space.w_dict)): + raise OperationError(space.w_TypeError, + space.wrap("argument after ** must be " + "a dictionary")) + keywords_w = [None] * space.int_w(space.len(w_starstararg)) + keywords = [None] * space.int_w(space.len(w_starstararg)) + i = 0 + for w_key in space.unpackiterable(w_starstararg): + try: + key = space.str_w(w_key) + except OperationError, e: + if not e.match(space, space.w_TypeError): + raise raise OperationError(space.w_TypeError, - space.wrap("argument after ** must be " - "a dictionary")) - keywords_w = [None] * space.int_w(space.len(w_starstararg)) - keywords = [None] * space.int_w(space.len(w_starstararg)) - i = 0 - for w_key in space.unpackiterable(w_starstararg): - try: - key = space.str_w(w_key) - except OperationError, e: - if not e.match(space, space.w_TypeError): - raise - raise OperationError(space.w_TypeError, - space.wrap("keywords must be strings")) - if self.keywords and key in self.keywords: - raise operationerrfmt(self.space.w_TypeError, - "got multiple values " - "for keyword argument " - "'%s'", key) - keywords[i] = key - keywords_w[i] = space.getitem(w_starstararg, w_key) - i += 1 - if self.keywords is None: - self.keywords = keywords - self.keywords_w = keywords_w - else: - self.keywords = self.keywords + keywords - self.keywords_w = self.keywords_w + keywords_w + space.wrap("keywords must be strings")) + if self.keywords and key in self.keywords: + raise operationerrfmt(self.space.w_TypeError, + "got multiple values " + "for keyword argument " + "'%s'", key) + keywords[i] = key + keywords_w[i] = space.getitem(w_starstararg, w_key) + i += 1 + if self.keywords is None: + self.keywords = keywords + self.keywords_w = keywords_w + else: + self.keywords = self.keywords + keywords + self.keywords_w = self.keywords_w + keywords_w def fixedunpack(self, argcount): """The simplest argument parsing: get the 'argcount' arguments, @@ -226,6 +233,10 @@ class Arguments(object): # argnames = list of formal parameter names # scope_w = resulting list of wrapped values # + + # some comments about the JIT: it assumes that signature is a constant, + # so all values coming from there can be assumed constant. It assumes + # that the length of the defaults_w does not vary too much. co_argcount = signature.num_argnames() # expected formal arguments, without */** has_vararg = signature.has_vararg() has_kwarg = signature.has_kwarg() @@ -245,12 +256,6 @@ class Arguments(object): args_w = self.arguments_w num_args = len(args_w) - keywords = self.keywords - keywords_w = self.keywords_w - num_kwds = 0 - if keywords is not None: - num_kwds = len(keywords) - avail = num_args + upfront if input_argcount < co_argcount: @@ -260,15 +265,24 @@ class Arguments(object): else: take = num_args + # letting the JIT unroll this loop is safe, because take is always + # smaller than co_argcount for i in range(take): scope_w[i + input_argcount] = args_w[i] input_argcount += take + keywords = self.keywords + keywords_w = self.keywords_w + num_kwds = 0 + if keywords is not None: + num_kwds = len(keywords) # the code assumes that keywords can potentially be large, but that # argnames is typically not too large num_remainingkwds = num_kwds used_keywords = None if keywords: + # letting JIT unroll the loop is *only* safe if the callsite didn't + # use **args because num_kwds can be arbitrarily large otherwise. used_keywords = [False] * num_kwds for i in range(num_kwds): name = keywords[i] @@ -276,7 +290,7 @@ class Arguments(object): if j < 0: continue elif j < input_argcount: - # check that no keyword argument conflicts with these note + # check that no keyword argument conflicts with these. note # that for this purpose we ignore the first blindargs, # which were put into place by prepend(). This way, # keywords do not conflict with the hidden extra argument diff --git a/pypy/interpreter/baseobjspace.py b/pypy/interpreter/baseobjspace.py index 1389ced080..18646a7bba 100644 --- a/pypy/interpreter/baseobjspace.py +++ b/pypy/interpreter/baseobjspace.py @@ -258,10 +258,9 @@ class ObjSpace(object): self.interned_strings = {} self.actionflag = ActionFlag() # changed by the signal module + self.check_signal_action = None # changed by the signal module self.user_del_action = UserDelAction(self) self.frame_trace_action = FrameTraceAction(self) - self.actionflag.register_action(self.user_del_action) - self.actionflag.register_action(self.frame_trace_action) from pypy.interpreter.pycode import cpython_magic, default_magic self.our_magic = default_magic @@ -299,8 +298,6 @@ class ObjSpace(object): self.timer.start("startup " + modname) mod.init(self) self.timer.stop("startup " + modname) - # Force the tick counter to have a valid value - self.actionflag.force_tick_counter() def finish(self): self.wait_for_thread_shutdown() diff --git a/pypy/interpreter/executioncontext.py b/pypy/interpreter/executioncontext.py index 409d3fab0c..fd34715d7d 100644 --- a/pypy/interpreter/executioncontext.py +++ b/pypy/interpreter/executioncontext.py @@ -5,6 +5,8 @@ from pypy.rlib.rarithmetic import LONG_BIT from pypy.rlib.unroll import unrolling_iterable from pypy.rlib import jit +TICK_COUNTER_STEP = 100 + def app_profile_call(space, w_callable, frame, event, w_arg): space.call_function(w_callable, space.wrap(frame), @@ -166,24 +168,19 @@ class ExecutionContext(object): if self.w_tracefunc is not None: self._trace(frame, 'return', w_retval) - def bytecode_trace(self, frame): + def bytecode_trace(self, frame, decr_by=TICK_COUNTER_STEP): "Trace function called before each bytecode." # this is split into a fast path and a slower path that is # not invoked every time bytecode_trace() is. actionflag = self.space.actionflag - ticker = actionflag.get() - if actionflag.has_bytecode_counter: # this "if" is constant-folded - ticker += 1 - actionflag.set(ticker) - if ticker & actionflag.interesting_bits: # fast check + if actionflag.decrement_ticker(decr_by) < 0: actionflag.action_dispatcher(self, frame) # slow path bytecode_trace._always_inline_ = True def bytecode_trace_after_exception(self, frame): "Like bytecode_trace(), but without increasing the ticker." actionflag = self.space.actionflag - ticker = actionflag.get() - if ticker & actionflag.interesting_bits: # fast check + if actionflag.get_ticker() < 0: actionflag.action_dispatcher(self, frame) # slow path bytecode_trace_after_exception._always_inline_ = True @@ -314,6 +311,13 @@ class ExecutionContext(object): frame.last_exception = last_exception self.is_tracing -= 1 + def checksignals(self): + """Similar to PyErr_CheckSignals(). If called in the main thread, + and if signals are pending for the process, deliver them now + (i.e. call the signal handlers).""" + if self.space.check_signal_action is not None: + self.space.check_signal_action.perform(self, None) + def _freeze_(self): raise Exception("ExecutionContext instances should not be seen during" " translation. Now is a good time to inspect the" @@ -321,118 +325,95 @@ class ExecutionContext(object): class AbstractActionFlag(object): - """This holds the global 'action flag'. It is a single bitfield - integer, with bits corresponding to AsyncAction objects that need to - be immediately triggered. The correspondance from bits to - AsyncAction instances is built at translation time. We can quickly - check if there is anything at all to do by checking if any of the - relevant bits is set. If threads are enabled, they consume the 20 - lower bits to hold a counter incremented at each bytecode, to know - when to release the GIL. + """This holds in an integer the 'ticker'. If threads are enabled, + it is decremented at each bytecode; when it reaches zero, we release + the GIL. And whether we have threads or not, it is forced to zero + whenever we fire any of the asynchronous actions. """ def __init__(self): self._periodic_actions = [] self._nonperiodic_actions = [] - self.unused_bits = self.FREE_BITS[:] self.has_bytecode_counter = False - self.interesting_bits = 0 + self.fired_actions = None + self.checkinterval_scaled = 100 * TICK_COUNTER_STEP self._rebuild_action_dispatcher() def fire(self, action): - """Request for the action to be run before the next opcode. - The action must have been registered at space initalization time.""" - ticker = self.get() - self.set(ticker | action.bitmask) - - def register_action(self, action): - "NOT_RPYTHON" - assert isinstance(action, AsyncAction) - if action.bitmask == 0: - while True: - action.bitmask = self.unused_bits.pop(0) - if not (action.bitmask & self.interesting_bits): - break - self.interesting_bits |= action.bitmask - if action.bitmask & self.BYTECODE_COUNTER_OVERFLOW_BIT: - assert action.bitmask == self.BYTECODE_COUNTER_OVERFLOW_BIT - self._periodic_actions.append(action) + """Request for the action to be run before the next opcode.""" + if not action._fired: + action._fired = True + if self.fired_actions is None: + self.fired_actions = [] + self.fired_actions.append(action) + # set the ticker to -1 in order to force action_dispatcher() + # to run at the next possible bytecode + self.reset_ticker(-1) + + def register_periodic_action(self, action, use_bytecode_counter): + """NOT_RPYTHON: + Register the PeriodicAsyncAction action to be called whenever the + tick counter becomes smaller than 0. If 'use_bytecode_counter' is + True, make sure that we decrease the tick counter at every bytecode. + This is needed for threads. Note that 'use_bytecode_counter' can be + False for signal handling, because whenever the process receives a + signal, the tick counter is set to -1 by C code in signals.h. + """ + assert isinstance(action, PeriodicAsyncAction) + self._periodic_actions.append(action) + if use_bytecode_counter: self.has_bytecode_counter = True - self.force_tick_counter() - else: - self._nonperiodic_actions.append((action, action.bitmask)) self._rebuild_action_dispatcher() - def setcheckinterval(self, space, interval): - if interval < self.CHECK_INTERVAL_MIN: - interval = self.CHECK_INTERVAL_MIN - elif interval > self.CHECK_INTERVAL_MAX: - interval = self.CHECK_INTERVAL_MAX - space.sys.checkinterval = interval - self.force_tick_counter() - - def force_tick_counter(self): - # force the tick counter to a valid value -- this actually forces - # it to reach BYTECODE_COUNTER_OVERFLOW_BIT at the next opcode. - ticker = self.get() - ticker &= ~ self.BYTECODE_COUNTER_OVERFLOW_BIT - ticker |= self.BYTECODE_COUNTER_MASK - self.set(ticker) + def getcheckinterval(self): + return self.checkinterval_scaled // TICK_COUNTER_STEP + + def setcheckinterval(self, interval): + MAX = sys.maxint // TICK_COUNTER_STEP + if interval < 1: + interval = 1 + elif interval > MAX: + interval = MAX + self.checkinterval_scaled = interval * TICK_COUNTER_STEP def _rebuild_action_dispatcher(self): periodic_actions = unrolling_iterable(self._periodic_actions) - nonperiodic_actions = unrolling_iterable(self._nonperiodic_actions) - has_bytecode_counter = self.has_bytecode_counter @jit.dont_look_inside def action_dispatcher(ec, frame): - # periodic actions - if has_bytecode_counter: - ticker = self.get() - if ticker & self.BYTECODE_COUNTER_OVERFLOW_BIT: - # We must run the periodic actions now, but first - # reset the bytecode counter (the following line - # works by assuming that we just overflowed the - # counter, i.e. BYTECODE_COUNTER_OVERFLOW_BIT is - # set but none of the BYTECODE_COUNTER_MASK bits - # are). - ticker -= ec.space.sys.checkinterval - self.set(ticker) - for action in periodic_actions: - action.perform(ec, frame) + # periodic actions (first reset the bytecode counter) + self.reset_ticker(self.checkinterval_scaled) + for action in periodic_actions: + action.perform(ec, frame) # nonperiodic actions - for action, bitmask in nonperiodic_actions: - ticker = self.get() - if ticker & bitmask: - self.set(ticker & ~ bitmask) + list = self.fired_actions + if list is not None: + self.fired_actions = None + for action in list: + action._fired = False action.perform(ec, frame) action_dispatcher._dont_inline_ = True self.action_dispatcher = action_dispatcher - # Bits reserved for the bytecode counter, if used - BYTECODE_COUNTER_MASK = (1 << 20) - 1 - BYTECODE_COUNTER_OVERFLOW_BIT = (1 << 20) - - # Free bits - FREE_BITS = [1 << _b for _b in range(21, LONG_BIT-1)] - - # The acceptable range of values for sys.checkinterval, so that - # the bytecode_counter fits in 20 bits - CHECK_INTERVAL_MIN = 1 - CHECK_INTERVAL_MAX = BYTECODE_COUNTER_OVERFLOW_BIT - class ActionFlag(AbstractActionFlag): """The normal class for space.actionflag. The signal module provides a different one.""" - _flags = 0 + _ticker = 0 + + def get_ticker(self): + return self._ticker - def get(self): - return self._flags + def reset_ticker(self, value): + self._ticker = value - def set(self, value): - self._flags = value + def decrement_ticker(self, by): + value = self._ticker + if self.has_bytecode_counter: # this 'if' is constant-folded + value -= by + self._ticker = value + return value class AsyncAction(object): @@ -440,7 +421,7 @@ class AsyncAction(object): asynchronously with regular bytecode execution, but that still need to occur between two opcodes, not at a completely random time. """ - bitmask = 0 # means 'please choose one bit automatically' + _fired = False def __init__(self, space): self.space = space @@ -453,10 +434,11 @@ class AsyncAction(object): def fire_after_thread_switch(self): """Bit of a hack: fire() the action but only the next time the GIL is released and re-acquired (i.e. after a potential thread switch). - Don't call this if threads are not enabled. + Don't call this if threads are not enabled. Currently limited to + one action (i.e. reserved for CheckSignalAction from module/signal). """ from pypy.module.thread.gil import spacestate - spacestate.set_actionflag_bit_after_thread_switch |= self.bitmask + spacestate.action_after_thread_switch = self def perform(self, executioncontext, frame): """To be overridden.""" @@ -466,7 +448,6 @@ class PeriodicAsyncAction(AsyncAction): """Abstract base class for actions that occur automatically every sys.checkinterval bytecodes. """ - bitmask = ActionFlag.BYTECODE_COUNTER_OVERFLOW_BIT class UserDelAction(AsyncAction): diff --git a/pypy/interpreter/pyopcode.py b/pypy/interpreter/pyopcode.py index 6a7542b460..ba19973e39 100644 --- a/pypy/interpreter/pyopcode.py +++ b/pypy/interpreter/pyopcode.py @@ -1140,7 +1140,7 @@ class SContinueLoop(SuspendedUnroller): state_pack_variables = staticmethod(state_pack_variables) -class FrameBlock: +class FrameBlock(object): """Abstract base class for frame blocks from the blockstack, used by the SETUP_XXX and POP_BLOCK opcodes.""" diff --git a/pypy/interpreter/test/test_argument.py b/pypy/interpreter/test/test_argument.py index 9d09c7c9a7..a17e6e214e 100644 --- a/pypy/interpreter/test/test_argument.py +++ b/pypy/interpreter/test/test_argument.py @@ -52,12 +52,17 @@ class TestSignature(object): assert y == "d" assert z == "e" +class dummy_wrapped_dict(dict): + def __nonzero__(self): + raise NotImplementedError class DummySpace(object): def newtuple(self, items): return tuple(items) def is_true(self, obj): + if isinstance(obj, dummy_wrapped_dict): + return bool(dict(obj)) return bool(obj) def fixedview(self, it): @@ -229,7 +234,7 @@ class TestArgumentsNormal(object): kwds_w = dict(kwds[:i]) keywords = kwds_w.keys() keywords_w = kwds_w.values() - w_kwds = dict(kwds[i:]) + w_kwds = dummy_wrapped_dict(kwds[i:]) if i == 2: w_kwds = None assert len(keywords) == len(keywords_w) @@ -265,7 +270,7 @@ class TestArgumentsNormal(object): kwds_w = dict(kwds[:i]) keywords = kwds_w.keys() keywords_w = kwds_w.values() - w_kwds = dict(kwds[i:]) + w_kwds = dummy_wrapped_dict(kwds[i:]) if i == 3: w_kwds = None args = Arguments(space, [1, 2], keywords, keywords_w, w_starstararg=w_kwds) diff --git a/pypy/interpreter/test/test_executioncontext.py b/pypy/interpreter/test/test_executioncontext.py index 78b7f09dad..3e0b4b11af 100644 --- a/pypy/interpreter/test/test_executioncontext.py +++ b/pypy/interpreter/test/test_executioncontext.py @@ -19,7 +19,6 @@ class TestExecutionContext: space = self.space a1 = DemoAction(space) - space.actionflag.register_action(a1) for i in range(20): # assert does not raise: space.appexec([], """(): @@ -50,7 +49,7 @@ class TestExecutionContext: space = self.space a2 = DemoAction(space) - space.actionflag.register_action(a2) + space.actionflag.register_periodic_action(a2, True) try: for i in range(500): space.appexec([], """(): @@ -59,7 +58,8 @@ class TestExecutionContext: """) except Finished: pass - assert space.sys.checkinterval / 10 < i < space.sys.checkinterval * 3 + checkinterval = space.actionflag.getcheckinterval() + assert checkinterval / 10 < i < checkinterval * 1.1 def test_llprofile(self): l = [] diff --git a/pypy/jit/backend/detect_cpu.py b/pypy/jit/backend/detect_cpu.py index ada7c3428f..9ea8e843bb 100644 --- a/pypy/jit/backend/detect_cpu.py +++ b/pypy/jit/backend/detect_cpu.py @@ -34,7 +34,17 @@ def autodetect_main_model(): 'x86_64': 'x86', }[mach] except KeyError: - raise ProcessorAutodetectError, "unsupported processor '%s'" % mach + return mach + +def autodetect_main_model_and_size(): + model = autodetect_main_model() + if sys.maxint == 2**31-1: + model += '_32' + elif sys.maxint == 2**63-1: + model += '_64' + else: + raise AssertionError, "bad value for sys.maxint" + return model def autodetect(): model = autodetect_main_model() diff --git a/pypy/jit/backend/llgraph/llimpl.py b/pypy/jit/backend/llgraph/llimpl.py index deefa7f57d..40e2aa2560 100644 --- a/pypy/jit/backend/llgraph/llimpl.py +++ b/pypy/jit/backend/llgraph/llimpl.py @@ -152,7 +152,7 @@ TYPES = { 'unicodegetitem' : (('ref', 'int'), 'int'), 'unicodesetitem' : (('ref', 'int', 'int'), 'int'), 'cast_ptr_to_int' : (('ref',), 'int'), - 'debug_merge_point': (('ref',), None), + 'debug_merge_point': (('ref', 'int'), None), 'force_token' : ((), 'int'), 'call_may_force' : (('int', 'varargs'), 'intorptr'), 'guard_not_forced': ((), None), @@ -568,7 +568,7 @@ class Frame(object): # return _op_default_implementation - def op_debug_merge_point(self, _, value): + def op_debug_merge_point(self, _, value, recdepth): from pypy.jit.metainterp.warmspot import get_stats loc = ConstPtr(value)._get_str() get_stats().add_merge_point_location(loc) diff --git a/pypy/jit/backend/llsupport/descr.py b/pypy/jit/backend/llsupport/descr.py index b5fdc24dff..be7d563e06 100644 --- a/pypy/jit/backend/llsupport/descr.py +++ b/pypy/jit/backend/llsupport/descr.py @@ -130,6 +130,7 @@ def get_field_descr(gccache, STRUCT, fieldname): # ArrayDescrs _A = lltype.GcArray(lltype.Signed) # a random gcarray +_AF = lltype.GcArray(lltype.Float) # an array of C doubles class BaseArrayDescr(AbstractDescr): @@ -171,16 +172,21 @@ class GcPtrArrayDescr(NonGcPtrArrayDescr): _clsname = 'GcPtrArrayDescr' _is_array_of_pointers = True -_CA = rffi.CArray(lltype.Signed) +class FloatArrayDescr(BaseArrayDescr): + _clsname = 'FloatArrayDescr' + _is_array_of_floats = True + def get_base_size(self, translate_support_code): + basesize, _, _ = symbolic.get_array_token(_AF, translate_support_code) + return basesize + def get_item_size(self, translate_support_code): + return symbolic.get_size(lltype.Float, translate_support_code) class BaseArrayNoLengthDescr(BaseArrayDescr): def get_base_size(self, translate_support_code): - basesize, _, _ = symbolic.get_array_token(_CA, translate_support_code) - return basesize + return 0 def get_ofs_length(self, translate_support_code): - _, _, ofslength = symbolic.get_array_token(_CA, translate_support_code) - return ofslength + return -1 class NonGcPtrArrayNoLengthDescr(BaseArrayNoLengthDescr): _clsname = 'NonGcPtrArrayNoLengthDescr' @@ -192,6 +198,8 @@ class GcPtrArrayNoLengthDescr(NonGcPtrArrayNoLengthDescr): _is_array_of_pointers = True def getArrayDescrClass(ARRAY): + if ARRAY.OF is lltype.Float: + return FloatArrayDescr return getDescrClass(ARRAY.OF, BaseArrayDescr, GcPtrArrayDescr, NonGcPtrArrayDescr, 'Array', 'get_item_size', '_is_array_of_floats', '_is_item_signed') @@ -219,7 +227,8 @@ def get_array_descr(gccache, ARRAY): basesize, itemsize, ofslength = symbolic.get_array_token(ARRAY, False) assert basesize == arraydescr.get_base_size(False) assert itemsize == arraydescr.get_item_size(False) - assert ofslength == arraydescr.get_ofs_length(False) + if not ARRAY._hints.get('nolength', False): + assert ofslength == arraydescr.get_ofs_length(False) if isinstance(ARRAY, lltype.GcArray): gccache.init_array_descr(ARRAY, arraydescr) cache[ARRAY] = arraydescr diff --git a/pypy/jit/backend/llsupport/gc.py b/pypy/jit/backend/llsupport/gc.py index 7a6e4b1274..ebce30a874 100644 --- a/pypy/jit/backend/llsupport/gc.py +++ b/pypy/jit/backend/llsupport/gc.py @@ -19,6 +19,7 @@ from pypy.jit.backend.llsupport.descr import get_call_descr # ____________________________________________________________ class GcLLDescription(GcCache): + minimal_size_in_nursery = 0 def __init__(self, gcdescr, translator=None, rtyper=None): GcCache.__init__(self, translator is not None, rtyper) self.gcdescr = gcdescr @@ -386,6 +387,7 @@ class GcLLDescr_framework(GcLLDescription): (self.array_basesize, _, self.array_length_ofs) = \ symbolic.get_array_token(lltype.GcArray(lltype.Signed), True) self.max_size_of_young_obj = self.GCClass.JIT_max_size_of_young_obj() + self.minimal_size_in_nursery=self.GCClass.JIT_minimal_size_in_nursery() # make a malloc function, with three arguments def malloc_basic(size, tid): @@ -468,6 +470,7 @@ class GcLLDescr_framework(GcLLDescription): def malloc_fixedsize_slowpath(size): if self.DEBUG: random_usage_of_xmm_registers() + assert size >= self.minimal_size_in_nursery try: gcref = llop1.do_malloc_fixedsize_clear(llmemory.GCREF, 0, size, True, False, False) @@ -570,6 +573,9 @@ class GcLLDescr_framework(GcLLDescription): # GETFIELD_RAW from the array 'gcrefs.list'. # newops = [] + # we can only remember one malloc since the next malloc can possibly + # collect + last_malloc = None for op in operations: if op.getopnum() == rop.DEBUG_MERGE_POINT: continue @@ -587,22 +593,32 @@ class GcLLDescr_framework(GcLLDescription): [ConstInt(addr)], box, self.single_gcref_descr)) op.setarg(i, box) + if op.is_malloc(): + last_malloc = op.result + elif op.can_malloc(): + last_malloc = None # ---------- write barrier for SETFIELD_GC ---------- if op.getopnum() == rop.SETFIELD_GC: - v = op.getarg(1) - if isinstance(v, BoxPtr) or (isinstance(v, ConstPtr) and - bool(v.value)): # store a non-NULL - self._gen_write_barrier(newops, op.getarg(0), v) - op = op.copy_and_change(rop.SETFIELD_RAW) + val = op.getarg(0) + # no need for a write barrier in the case of previous malloc + if val is not last_malloc: + v = op.getarg(1) + if isinstance(v, BoxPtr) or (isinstance(v, ConstPtr) and + bool(v.value)): # store a non-NULL + self._gen_write_barrier(newops, op.getarg(0), v) + op = op.copy_and_change(rop.SETFIELD_RAW) # ---------- write barrier for SETARRAYITEM_GC ---------- if op.getopnum() == rop.SETARRAYITEM_GC: - v = op.getarg(2) - if isinstance(v, BoxPtr) or (isinstance(v, ConstPtr) and - bool(v.value)): # store a non-NULL - # XXX detect when we should produce a - # write_barrier_from_array - self._gen_write_barrier(newops, op.getarg(0), v) - op = op.copy_and_change(rop.SETARRAYITEM_RAW) + val = op.getarg(0) + # no need for a write barrier in the case of previous malloc + if val is not last_malloc: + v = op.getarg(2) + if isinstance(v, BoxPtr) or (isinstance(v, ConstPtr) and + bool(v.value)): # store a non-NULL + # XXX detect when we should produce a + # write_barrier_from_array + self._gen_write_barrier(newops, op.getarg(0), v) + op = op.copy_and_change(rop.SETARRAYITEM_RAW) # ---------- newops.append(op) del operations[:] diff --git a/pypy/jit/backend/llsupport/test/test_descr.py b/pypy/jit/backend/llsupport/test/test_descr.py index 23a1991a5a..32d75c0637 100644 --- a/pypy/jit/backend/llsupport/test/test_descr.py +++ b/pypy/jit/backend/llsupport/test/test_descr.py @@ -5,6 +5,7 @@ from pypy.rlib.objectmodel import Symbolic from pypy.rpython.annlowlevel import llhelper from pypy.jit.metainterp.history import BoxInt, BoxFloat, BoxPtr from pypy.jit.metainterp import history +import struct def test_get_size_descr(): c0 = GcCache(False) @@ -130,11 +131,13 @@ def test_get_array_descr(): assert not descr3.is_array_of_floats() assert descr4.is_array_of_floats() # - WORD = rffi.sizeof(lltype.Signed) - assert descr1.get_base_size(False) == WORD - assert descr2.get_base_size(False) == WORD - assert descr3.get_base_size(False) == WORD - assert descr4.get_base_size(False) == WORD + def get_alignment(code): + # Retrieve default alignment for the compiler/platform + return struct.calcsize('l' + code) - struct.calcsize(code) + assert descr1.get_base_size(False) == get_alignment('c') + assert descr2.get_base_size(False) == get_alignment('p') + assert descr3.get_base_size(False) == get_alignment('p') + assert descr4.get_base_size(False) == get_alignment('d') assert descr1.get_ofs_length(False) == 0 assert descr2.get_ofs_length(False) == 0 assert descr3.get_ofs_length(False) == 0 diff --git a/pypy/jit/backend/llsupport/test/test_gc.py b/pypy/jit/backend/llsupport/test/test_gc.py index 8fc2dbb399..afd0e67275 100644 --- a/pypy/jit/backend/llsupport/test/test_gc.py +++ b/pypy/jit/backend/llsupport/test/test_gc.py @@ -6,7 +6,9 @@ from pypy.jit.backend.llsupport.descr import * from pypy.jit.backend.llsupport.gc import * from pypy.jit.backend.llsupport import symbolic from pypy.jit.metainterp.gc import get_description - +from pypy.jit.tool.oparser import parse +from pypy.rpython.lltypesystem.rclass import OBJECT, OBJECT_VTABLE +from pypy.jit.metainterp.test.test_optimizeopt import equaloplists def test_boehm(): gc_ll_descr = GcLLDescr_boehm(None, None, None) @@ -114,7 +116,7 @@ def test_GcRootMap_asmgcc(): assert gcrootmap._gcmap[i*2+1] == expected_shapeaddr[i] -class FakeLLOp: +class FakeLLOp(object): def __init__(self): self.record = [] @@ -148,19 +150,19 @@ class FakeLLOp: return llhelper(FPTRTYPE, self._write_barrier_failing_case) -class TestFramework: +class TestFramework(object): gc = 'hybrid' def setup_method(self, meth): - class config_: - class translation: + class config_(object): + class translation(object): gc = self.gc gcrootfinder = 'asmgcc' gctransformer = 'framework' gcremovetypeptr = False - class FakeTranslator: + class FakeTranslator(object): config = config_ - class FakeCPU: + class FakeCPU(object): def cast_adr_to_int(self, adr): ptr = llmemory.cast_adr_to_ptr(adr, gc_ll_descr.WB_FUNCPTR) assert ptr._obj._callable == llop1._write_barrier_failing_case @@ -270,7 +272,7 @@ class TestFramework: def test_get_rid_of_debug_merge_point(self): operations = [ - ResOperation(rop.DEBUG_MERGE_POINT, ['dummy'], None), + ResOperation(rop.DEBUG_MERGE_POINT, ['dummy', 2], None), ] gc_ll_descr = self.gc_ll_descr gc_ll_descr.rewrite_assembler(None, operations) @@ -278,11 +280,11 @@ class TestFramework: def test_rewrite_assembler_1(self): # check rewriting of ConstPtrs - class MyFakeCPU: + class MyFakeCPU(object): def cast_adr_to_int(self, adr): assert adr == "some fake address" return 43 - class MyFakeGCRefList: + class MyFakeGCRefList(object): def get_address_of_gcref(self, s_gcref1): assert s_gcref1 == s_gcref return "some fake address" @@ -311,10 +313,10 @@ class TestFramework: def test_rewrite_assembler_1_cannot_move(self): # check rewriting of ConstPtrs - class MyFakeCPU: + class MyFakeCPU(object): def cast_adr_to_int(self, adr): xxx # should not be called - class MyFakeGCRefList: + class MyFakeGCRefList(object): def get_address_of_gcref(self, s_gcref1): seen.append(s_gcref1) assert s_gcref1 == s_gcref @@ -394,6 +396,68 @@ class TestFramework: assert operations[1].getarg(2) == v_value assert operations[1].getdescr() == array_descr + def test_rewrite_assembler_initialization_store(self): + S = lltype.GcStruct('S', ('parent', OBJECT), + ('x', lltype.Signed)) + s_vtable = lltype.malloc(OBJECT_VTABLE, immortal=True) + xdescr = get_field_descr(self.gc_ll_descr, S, 'x') + ops = parse(""" + [p1] + p0 = new_with_vtable(ConstClass(s_vtable)) + setfield_gc(p0, p1, descr=xdescr) + jump() + """, namespace=locals()) + expected = parse(""" + [p1] + p0 = new_with_vtable(ConstClass(s_vtable)) + # no write barrier + setfield_gc(p0, p1, descr=xdescr) + jump() + """, namespace=locals()) + self.gc_ll_descr.rewrite_assembler(self.fake_cpu, ops.operations) + equaloplists(ops.operations, expected.operations) + + def test_rewrite_assembler_initialization_store_2(self): + S = lltype.GcStruct('S', ('parent', OBJECT), + ('x', lltype.Signed)) + s_vtable = lltype.malloc(OBJECT_VTABLE, immortal=True) + wbdescr = self.gc_ll_descr.write_barrier_descr + xdescr = get_field_descr(self.gc_ll_descr, S, 'x') + ops = parse(""" + [p1] + p0 = new_with_vtable(ConstClass(s_vtable)) + p3 = new_with_vtable(ConstClass(s_vtable)) + setfield_gc(p0, p1, descr=xdescr) + jump() + """, namespace=locals()) + expected = parse(""" + [p1] + p0 = new_with_vtable(ConstClass(s_vtable)) + p3 = new_with_vtable(ConstClass(s_vtable)) + cond_call_gc_wb(p0, p1, descr=wbdescr) + setfield_raw(p0, p1, descr=xdescr) + jump() + """, namespace=locals()) + self.gc_ll_descr.rewrite_assembler(self.fake_cpu, ops.operations) + equaloplists(ops.operations, expected.operations) + + def test_rewrite_assembler_initialization_store_3(self): + A = lltype.GcArray(lltype.Ptr(lltype.GcStruct('S'))) + arraydescr = get_array_descr(self.gc_ll_descr, A) + ops = parse(""" + [p1] + p0 = new_array(3, descr=arraydescr) + setarrayitem_gc(p0, 0, p1, descr=arraydescr) + jump() + """, namespace=locals()) + expected = parse(""" + [p1] + p0 = new_array(3, descr=arraydescr) + setarrayitem_gc(p0, 0, p1, descr=arraydescr) + jump() + """, namespace=locals()) + self.gc_ll_descr.rewrite_assembler(self.fake_cpu, ops.operations) + equaloplists(ops.operations, expected.operations) class TestFrameworkMiniMark(TestFramework): gc = 'minimark' diff --git a/pypy/jit/backend/test/runner_test.py b/pypy/jit/backend/test/runner_test.py index 95d7e3bf5a..9c437fbf00 100644 --- a/pypy/jit/backend/test/runner_test.py +++ b/pypy/jit/backend/test/runner_test.py @@ -2168,12 +2168,14 @@ class LLtypeBackendTest(BaseBackendTest): # Tested with a function that intentionally does not cast the # result to RESTYPE, but makes sure that we return the whole # value in eax or rax. - eci = ExternalCompilationInfo(separate_module_sources=[""" + eci = ExternalCompilationInfo( + separate_module_sources=[""" long fn_test_result_of_call(long x) { return x + 1; } - """]) + """], + export_symbols=['fn_test_result_of_call']) f = rffi.llexternal('fn_test_result_of_call', [lltype.Signed], RESTYPE, compilation_info=eci, _nowrapper=True) value = intmask(0xFFEEDDCCBBAA9988) @@ -2199,12 +2201,14 @@ class LLtypeBackendTest(BaseBackendTest): # Tested with a function that intentionally does not cast the # result to RESTYPE, but makes sure that we return the whole # value in eax or rax. - eci = ExternalCompilationInfo(separate_module_sources=[""" + eci = ExternalCompilationInfo( + separate_module_sources=[""" long fn_test_result_of_call(long x) { return x + 1; } - """]) + """], + export_symbols=['fn_test_result_of_call']) f = rffi.llexternal('fn_test_result_of_call', [lltype.Signed], RESTYPE, compilation_info=eci, _nowrapper=True) value = intmask(0xFFEEDDCCBBAA9988) diff --git a/pypy/jit/backend/x86/assembler.py b/pypy/jit/backend/x86/assembler.py index 46caebe614..46e430ef72 100644 --- a/pypy/jit/backend/x86/assembler.py +++ b/pypy/jit/backend/x86/assembler.py @@ -303,6 +303,7 @@ class Assembler386(object): _x86_frame_depth _x86_param_depth _x86_arglocs + _x86_debug_checksum """ if not we_are_translated(): # Arguments should be unique @@ -312,7 +313,7 @@ class Assembler386(object): funcname = self._find_debug_merge_point(operations) if log: self._register_counter() - operations = self._inject_debugging_code(operations) + operations = self._inject_debugging_code(looptoken, operations) regalloc = RegAlloc(self, self.cpu.translate_support_code) arglocs = regalloc.prepare_loop(inputargs, operations, looptoken) @@ -336,8 +337,9 @@ class Assembler386(object): self._assemble_bootstrap_direct_call(arglocs, curadr, frame_depth+param_depth) # - debug_print("Loop #", looptoken.number, "has address", - looptoken._x86_loop_code, "to", self.mc.tell()) + debug_print("Loop #%d has address %x to %x" % (looptoken.number, + looptoken._x86_loop_code, + self.mc.tell())) self.mc.end_function() self.write_pending_failure_recoveries() @@ -350,7 +352,7 @@ class Assembler386(object): funcname = self._find_debug_merge_point(operations) if log: self._register_counter() - operations = self._inject_debugging_code(operations) + operations = self._inject_debugging_code(faildescr, operations) arglocs = self.rebuild_faillocs_from_descr( faildescr._x86_failure_recovery_bytecode) @@ -358,7 +360,6 @@ class Assembler386(object): assert ([loc.assembler() for loc in arglocs] == [loc.assembler() for loc in faildescr._x86_debug_faillocs]) regalloc = RegAlloc(self, self.cpu.translate_support_code) - operations = self._inject_debugging_code(operations) fail_depths = faildescr._x86_current_depths regalloc.prepare_bridge(fail_depths, inputargs, arglocs, operations) @@ -378,9 +379,8 @@ class Assembler386(object): faildescr._x86_bridge_param_depth = param_depth # patch the jump from original guard self.patch_jump_for_descr(faildescr, adr_bridge) - debug_print("Bridge out of guard", - descr_number, - "has address", adr_bridge, "to", self.mc.tell()) + debug_print("Bridge out of guard %d has address %x to %x" % + (descr_number, adr_bridge, self.mc.tell())) self.mc.end_function() self.write_pending_failure_recoveries() @@ -433,9 +433,14 @@ class Assembler386(object): mc.done() - def _inject_debugging_code(self, operations): + @specialize.argtype(1) + def _inject_debugging_code(self, looptoken, operations): if self._debug: # before doing anything, let's increase a counter + s = 0 + for op in operations: + s += op.getopnum() + looptoken._x86_debug_checksum = s c_adr = ConstInt(rffi.cast(lltype.Signed, self.loop_run_counters[-1][1])) box = BoxInt() @@ -1768,13 +1773,18 @@ class Assembler386(object): self.implement_guard(guard_token, 'L') def genop_discard_cond_call_gc_wb(self, op, arglocs): - # use 'mc._mc' directly instead of 'mc', to avoid - # bad surprizes if the code buffer is mostly full + # Write code equivalent to write_barrier() in the GC: it checks + # a flag in the object at arglocs[0], and if set, it calls the + # function remember_young_pointer() from the GC. The two arguments + # to the call are in arglocs[:2]. The rest, arglocs[2:], contains + # registers that need to be saved and restored across the call. descr = op.getdescr() if we_are_translated(): cls = self.cpu.gc_ll_descr.has_write_barrier_class() assert cls is not None and isinstance(descr, cls) loc_base = arglocs[0] + # ensure that enough bytes are available to write the whole + # following piece of code atomically (for the JZ) self.mc.ensure_bytes_available(256) self.mc.TEST8_mi((loc_base.value, descr.jit_wb_if_flag_byteofs), descr.jit_wb_if_flag_singlebyte) @@ -1782,36 +1792,39 @@ class Assembler386(object): jz_location = self.mc.get_relative_pos() # the following is supposed to be the slow path, so whenever possible # we choose the most compact encoding over the most efficient one. - # XXX improve a bit, particularly for IS_X86_64. - for i in range(len(arglocs)-1, -1, -1): + if IS_X86_32: + limit = -1 # push all arglocs on the stack + elif IS_X86_64: + limit = 1 # push only arglocs[2:] on the stack + for i in range(len(arglocs)-1, limit, -1): loc = arglocs[i] if isinstance(loc, RegLoc): self.mc.PUSH_r(loc.value) else: - if IS_X86_64: - self.mc.MOV_ri(X86_64_SCRATCH_REG.value, loc.getint()) - self.mc.PUSH_r(X86_64_SCRATCH_REG.value) - else: - self.mc.PUSH_i32(loc.getint()) - + assert not IS_X86_64 # there should only be regs in arglocs[2:] + self.mc.PUSH_i32(loc.getint()) if IS_X86_64: # We clobber these registers to pass the arguments, but that's # okay, because consider_cond_call_gc_wb makes sure that any - # caller-save registers with values in them are present in arglocs, - # so they are saved on the stack above and restored below - self.mc.MOV_rs(edi.value, 0) - self.mc.MOV_rs(esi.value, 8) + # caller-save registers with values in them are present in + # arglocs[2:] too, so they are saved on the stack above and + # restored below. + remap_frame_layout(self, arglocs[:2], [edi, esi], + X86_64_SCRATCH_REG) # misaligned stack in the call, but it's ok because the write barrier # is not going to call anything more. Also, this assumes that the - # write barrier does not touch the xmm registers. + # write barrier does not touch the xmm registers. (Slightly delicate + # assumption, given that the write barrier can end up calling the + # platform's malloc() from AddressStack.append(). XXX may need to + # be done properly) self.mc.CALL(imm(descr.get_write_barrier_fn(self.cpu))) - for i in range(len(arglocs)): + if IS_X86_32: + self.mc.ADD_ri(esp.value, 2*WORD) + for i in range(2, len(arglocs)): loc = arglocs[i] - if isinstance(loc, RegLoc): - self.mc.POP_r(loc.value) - else: - self.mc.ADD_ri(esp.value, WORD) # ignore the pushed constant + assert isinstance(loc, RegLoc) + self.mc.POP_r(loc.value) # patch the JZ above offset = self.mc.get_relative_pos() - jz_location assert 0 < offset <= 127 @@ -1848,6 +1861,7 @@ class Assembler386(object): def malloc_cond_fixedsize(self, nursery_free_adr, nursery_top_adr, size, tid): + size = max(size, self.cpu.gc_ll_descr.minimal_size_in_nursery) self.mc.ensure_bytes_available(256) self.mc.MOV(eax, heap(nursery_free_adr)) self.mc.LEA_rm(edx.value, (eax.value, size)) diff --git a/pypy/jit/backend/x86/test/test_runner.py b/pypy/jit/backend/x86/test/test_runner.py index c307e515bd..05e21d5056 100644 --- a/pypy/jit/backend/x86/test/test_runner.py +++ b/pypy/jit/backend/x86/test/test_runner.py @@ -346,7 +346,7 @@ class TestX86(LLtypeBackendTest): return self.val operations = [ - ResOperation(rop.DEBUG_MERGE_POINT, [FakeString("hello")], None), + ResOperation(rop.DEBUG_MERGE_POINT, [FakeString("hello"), 0], None), ResOperation(rop.INT_ADD, [i0, ConstInt(1)], i1), ResOperation(rop.INT_LE, [i1, ConstInt(9)], i2), ResOperation(rop.GUARD_TRUE, [i2], None, descr=faildescr1), @@ -365,7 +365,7 @@ class TestX86(LLtypeBackendTest): bridge = [ ResOperation(rop.INT_LE, [i1b, ConstInt(19)], i3), ResOperation(rop.GUARD_TRUE, [i3], None, descr=faildescr2), - ResOperation(rop.DEBUG_MERGE_POINT, [FakeString("bye")], None), + ResOperation(rop.DEBUG_MERGE_POINT, [FakeString("bye"), 0], None), ResOperation(rop.JUMP, [i1b], None, descr=looptoken), ] bridge[1].setfailargs([i1b]) @@ -497,7 +497,7 @@ class TestDebuggingAssembler(object): def test_debugger_on(self): loop = """ [i0] - debug_merge_point('xyz') + debug_merge_point('xyz', 0) i1 = int_add(i0, 1) i2 = int_ge(i1, 10) guard_false(i2) [] @@ -515,3 +515,20 @@ class TestDebuggingAssembler(object): self.cpu.finish_once() lines = py.path.local(self.logfile + ".count").readlines() assert lines[0] == '0:10\n' # '10 xyz\n' + + def test_debugger_checksum(self): + loop = """ + [i0] + debug_merge_point('xyz', 0) + i1 = int_add(i0, 1) + i2 = int_ge(i1, 10) + guard_false(i2) [] + jump(i1) + """ + ops = parse(loop) + self.cpu.assembler.set_debug(True) + self.cpu.compile_loop(ops.inputargs, ops.operations, ops.token) + self.cpu.set_future_value_int(0, 0) + self.cpu.execute_token(ops.token) + assert ops.token._x86_debug_checksum == sum([op.getopnum() + for op in ops.operations]) diff --git a/pypy/jit/backend/x86/test/test_zrpy_gc.py b/pypy/jit/backend/x86/test/test_zrpy_gc.py index 5fe1cfdb98..0fd525b09b 100644 --- a/pypy/jit/backend/x86/test/test_zrpy_gc.py +++ b/pypy/jit/backend/x86/test/test_zrpy_gc.py @@ -550,3 +550,29 @@ class TestCompileFramework(object): def test_compile_framework_float(self): self.run('compile_framework_float') + + def define_compile_framework_minimal_size_in_nursery(self): + S = lltype.GcStruct('S') # no fields! + T = lltype.GcStruct('T', ('i', lltype.Signed)) + @unroll_safe + def f42(n, x, x0, x1, x2, x3, x4, x5, x6, x7, l, s): + lst1 = [] + lst2 = [] + i = 0 + while i < 42: + s1 = lltype.malloc(S) + t1 = lltype.malloc(T) + t1.i = 10000 + i + n + lst1.append(s1) + lst2.append(t1) + i += 1 + i = 0 + while i < 42: + check(lst2[i].i == 10000 + i + n) + i += 1 + n -= 1 + return n, x, x0, x1, x2, x3, x4, x5, x6, x7, l, s + return None, f42, None + + def test_compile_framework_minimal_size_in_nursery(self): + self.run('compile_framework_minimal_size_in_nursery') diff --git a/pypy/jit/backend/x86/test/test_ztranslation.py b/pypy/jit/backend/x86/test/test_ztranslation.py index f3510639fd..1e3f81b553 100644 --- a/pypy/jit/backend/x86/test/test_ztranslation.py +++ b/pypy/jit/backend/x86/test/test_ztranslation.py @@ -2,6 +2,7 @@ import py, os, sys from pypy.tool.udir import udir from pypy.rlib.jit import JitDriver, OPTIMIZER_FULL, unroll_parameters from pypy.rlib.jit import PARAMETERS, dont_look_inside +from pypy.rlib.jit import hint from pypy.jit.metainterp.jitprof import Profiler from pypy.jit.backend.detect_cpu import getcpuclass from pypy.jit.backend.test.support import CCompiledMixin @@ -9,6 +10,7 @@ from pypy.jit.codewriter.policy import StopAtXPolicy from pypy.translator.translator import TranslationContext from pypy.jit.backend.x86.arch import IS_X86_32, IS_X86_64 from pypy.config.translationoption import DEFL_GC +from pypy.rlib import rgc class TestTranslationX86(CCompiledMixin): CPUClass = getcpuclass() @@ -82,12 +84,12 @@ class TestTranslationX86(CCompiledMixin): argchain.arg(x) res = func.call(argchain, rffi.DOUBLE) i -= 1 - return res + return int(res) # def main(i, j): return f(i, j) + libffi_stuff(i, j) - expected = f(40, -49) - res = self.meta_interp(f, [40, -49]) + expected = main(40, -49) + res = self.meta_interp(main, [40, -49]) assert res == expected def test_direct_assembler_call_translates(self): diff --git a/pypy/jit/backend/x86/tool/viewcode.py b/pypy/jit/backend/x86/tool/viewcode.py index 33a1895121..dd0ff1606f 100755 --- a/pypy/jit/backend/x86/tool/viewcode.py +++ b/pypy/jit/backend/x86/tool/viewcode.py @@ -37,7 +37,7 @@ def machine_code_dump(data, originaddr, backend_name): 'x86_64': 'x86-64', 'i386': 'i386', } - objdump = ('objdump -M intel,%(backend)s -b binary -m i386 ' + objdump = ('objdump -M %(backend)s -b binary -m i386 ' '--adjust-vma=%(origin)d -D %(file)s') # f = open(tmpfile, 'wb') diff --git a/pypy/jit/codewriter/assembler.py b/pypy/jit/codewriter/assembler.py index 0fb4f6051a..1fdd0d6ea2 100644 --- a/pypy/jit/codewriter/assembler.py +++ b/pypy/jit/codewriter/assembler.py @@ -233,10 +233,9 @@ class Assembler(object): addr = llmemory.cast_ptr_to_adr(value) self.list_of_addr2name.append((addr, name)) - def finished(self): + def finished(self, callinfocollection): # Helper called at the end of assembling. Registers the extra # functions shown in _callinfo_for_oopspec. - from pypy.jit.codewriter.effectinfo import _callinfo_for_oopspec - for _, func in _callinfo_for_oopspec.values(): + for func in callinfocollection.all_function_addresses_as_int(): func = heaptracker.int2adr(func) self.see_raw_object(func.ptr) diff --git a/pypy/jit/codewriter/call.py b/pypy/jit/codewriter/call.py index 9dc12a7cb7..2f82e49a08 100644 --- a/pypy/jit/codewriter/call.py +++ b/pypy/jit/codewriter/call.py @@ -7,7 +7,7 @@ from pypy.jit.codewriter import support from pypy.jit.codewriter.jitcode import JitCode from pypy.jit.codewriter.effectinfo import VirtualizableAnalyzer from pypy.jit.codewriter.effectinfo import effectinfo_from_writeanalyze -from pypy.jit.codewriter.effectinfo import EffectInfo +from pypy.jit.codewriter.effectinfo import EffectInfo, CallInfoCollection from pypy.translator.simplify import get_funcobj, get_functype from pypy.rpython.lltypesystem import lltype, llmemory from pypy.translator.backendopt.canraise import RaiseAnalyzer @@ -23,6 +23,7 @@ class CallControl(object): self.jitdrivers_sd = jitdrivers_sd self.jitcodes = {} # map {graph: jitcode} self.unfinished_graphs = [] # list of graphs with pending jitcodes + self.callinfocollection = CallInfoCollection() if hasattr(cpu, 'rtyper'): # for tests self.rtyper = cpu.rtyper translator = self.rtyper.annotator.translator diff --git a/pypy/jit/codewriter/codewriter.py b/pypy/jit/codewriter/codewriter.py index 9013ed9615..ff2c860c17 100644 --- a/pypy/jit/codewriter/codewriter.py +++ b/pypy/jit/codewriter/codewriter.py @@ -73,7 +73,7 @@ class CodeWriter(object): count += 1 if not count % 500: log.info("Produced %d jitcodes" % count) - self.assembler.finished() + self.assembler.finished(self.callcontrol.callinfocollection) heaptracker.finish_registering(self.cpu) log.info("there are %d JitCode instances." % count) diff --git a/pypy/jit/codewriter/effectinfo.py b/pypy/jit/codewriter/effectinfo.py index d8c18af704..84c7edcab2 100644 --- a/pypy/jit/codewriter/effectinfo.py +++ b/pypy/jit/codewriter/effectinfo.py @@ -144,30 +144,44 @@ class VirtualizableAnalyzer(BoolGraphAnalyzer): # ____________________________________________________________ -_callinfo_for_oopspec = {} # {oopspecindex: (calldescr, func_as_int)} - -def callinfo_for_oopspec(oopspecindex): - """A function that returns the calldescr and the function - address (as an int) of one of the OS_XYZ functions defined above. - Don't use this if there might be several implementations of the same - OS_XYZ specialized by type, e.g. OS_ARRAYCOPY.""" - try: - return _callinfo_for_oopspec[oopspecindex] - except KeyError: - return (None, 0) - - -def _funcptr_for_oopspec_memo(oopspecindex): - from pypy.jit.codewriter import heaptracker - _, func_as_int = callinfo_for_oopspec(oopspecindex) - funcadr = heaptracker.int2adr(func_as_int) - return funcadr.ptr -_funcptr_for_oopspec_memo._annspecialcase_ = 'specialize:memo' - -def funcptr_for_oopspec(oopspecindex): - """A memo function that returns a pointer to the function described - by OS_XYZ (as a real low-level function pointer).""" - funcptr = _funcptr_for_oopspec_memo(oopspecindex) - assert funcptr - return funcptr -funcptr_for_oopspec._annspecialcase_ = 'specialize:arg(0)' +class CallInfoCollection(object): + def __init__(self): + # {oopspecindex: (calldescr, func_as_int)} + self._callinfo_for_oopspec = {} + + def _freeze_(self): + return True + + def add(self, oopspecindex, calldescr, func_as_int): + self._callinfo_for_oopspec[oopspecindex] = calldescr, func_as_int + + def has_oopspec(self, oopspecindex): + return oopspecindex in self._callinfo_for_oopspec + + def all_function_addresses_as_int(self): + return [func for (_, func) in self._callinfo_for_oopspec.values()] + + def callinfo_for_oopspec(self, oopspecindex): + """A function that returns the calldescr and the function + address (as an int) of one of the OS_XYZ functions defined above. + Don't use this if there might be several implementations of the same + OS_XYZ specialized by type, e.g. OS_ARRAYCOPY.""" + try: + return self._callinfo_for_oopspec[oopspecindex] + except KeyError: + return (None, 0) + + def _funcptr_for_oopspec_memo(self, oopspecindex): + from pypy.jit.codewriter import heaptracker + _, func_as_int = self.callinfo_for_oopspec(oopspecindex) + funcadr = heaptracker.int2adr(func_as_int) + return funcadr.ptr + _funcptr_for_oopspec_memo._annspecialcase_ = 'specialize:memo' + + def funcptr_for_oopspec(self, oopspecindex): + """A memo function that returns a pointer to the function described + by OS_XYZ (as a real low-level function pointer).""" + funcptr = self._funcptr_for_oopspec_memo(oopspecindex) + assert funcptr + return funcptr + funcptr_for_oopspec._annspecialcase_ = 'specialize:arg(1)' diff --git a/pypy/jit/codewriter/jtransform.py b/pypy/jit/codewriter/jtransform.py index 326e1fe16f..68e7127e02 100644 --- a/pypy/jit/codewriter/jtransform.py +++ b/pypy/jit/codewriter/jtransform.py @@ -6,7 +6,7 @@ from pypy.objspace.flow.model import SpaceOperation, Variable, Constant from pypy.objspace.flow.model import Block, Link, c_last_exception from pypy.jit.codewriter.flatten import ListOfKind, IndirectCallTargets from pypy.jit.codewriter import support, heaptracker -from pypy.jit.codewriter.effectinfo import EffectInfo, _callinfo_for_oopspec +from pypy.jit.codewriter.effectinfo import EffectInfo from pypy.jit.codewriter.policy import log from pypy.jit.metainterp.typesystem import deref, arrayItem from pypy.rlib import objectmodel @@ -873,6 +873,8 @@ class Transformer(object): elif oopspec_name == 'jit.assert_green': kind = getkind(args[0].concretetype) return SpaceOperation('%s_assert_green' % kind, args, None) + elif oopspec_name == 'jit.current_trace_length': + return SpaceOperation('current_trace_length', [], op.result) else: raise AssertionError("missing support for %r" % oopspec_name) @@ -1082,7 +1084,8 @@ class Transformer(object): else: func = heaptracker.adr2int( llmemory.cast_ptr_to_adr(op.args[0].value)) - _callinfo_for_oopspec[oopspecindex] = calldescr, func + self.callcontrol.callinfocollection.add(oopspecindex, + calldescr, func) op1 = self.rewrite_call(op, 'residual_call', [op.args[0], calldescr], args=args) @@ -1093,7 +1096,7 @@ class Transformer(object): def _register_extra_helper(self, oopspecindex, oopspec_name, argtypes, resulttype): # a bit hackish - if oopspecindex in _callinfo_for_oopspec: + if self.callcontrol.callinfocollection.has_oopspec(oopspecindex): return c_func, TP = support.builtin_func_for_spec(self.cpu.rtyper, oopspec_name, argtypes, @@ -1107,7 +1110,7 @@ class Transformer(object): else: func = heaptracker.adr2int( llmemory.cast_ptr_to_adr(c_func.value)) - _callinfo_for_oopspec[oopspecindex] = calldescr, func + self.callcontrol.callinfocollection.add(oopspecindex, calldescr, func) def _handle_stroruni_call(self, op, oopspec_name, args): SoU = args[0].concretetype # Ptr(STR) or Ptr(UNICODE) diff --git a/pypy/jit/codewriter/test/test_jtransform.py b/pypy/jit/codewriter/test/test_jtransform.py index 760feea625..01dca13392 100644 --- a/pypy/jit/codewriter/test/test_jtransform.py +++ b/pypy/jit/codewriter/test/test_jtransform.py @@ -74,7 +74,20 @@ class FakeRegularIndirectCallControl: def calldescr_canraise(self, calldescr): return False +class FakeCallInfoCollection: + def __init__(self): + self.seen = [] + def add(self, oopspecindex, calldescr, func): + self.seen.append((oopspecindex, calldescr, func)) + def has_oopspec(self, oopspecindex): + for i, c, f in self.seen: + if i == oopspecindex: + return True + return False + class FakeBuiltinCallControl: + def __init__(self): + self.callinfocollection = FakeCallInfoCollection() def guess_call_kind(self, op): return 'builtin' def getcalldescr(self, op, oopspecindex=None): @@ -810,7 +823,8 @@ def test_unicode_concat(): v2 = varoftype(PSTR) v3 = varoftype(PSTR) op = SpaceOperation('direct_call', [const(func), v1, v2], v3) - tr = Transformer(FakeCPU(), FakeBuiltinCallControl()) + cc = FakeBuiltinCallControl() + tr = Transformer(FakeCPU(), cc) op1 = tr.rewrite_operation(op) assert op1.opname == 'residual_call_r_r' assert op1.args[0].value == func @@ -819,9 +833,10 @@ def test_unicode_concat(): assert op1.result == v3 # # check the callinfo_for_oopspec - got = effectinfo.callinfo_for_oopspec(effectinfo.EffectInfo.OS_UNI_CONCAT) - assert got[0] == op1.args[1] # the calldescr - assert heaptracker.int2adr(got[1]) == llmemory.cast_ptr_to_adr(func) + got = cc.callinfocollection.seen[0] + assert got[0] == effectinfo.EffectInfo.OS_UNI_CONCAT + assert got[1] == op1.args[1] # the calldescr + assert heaptracker.int2adr(got[2]) == llmemory.cast_ptr_to_adr(func) def test_str_slice(): # test that the oopspec is present and correctly transformed @@ -893,7 +908,8 @@ def test_unicode_eq_checknull_char(): v2 = varoftype(PUNICODE) v3 = varoftype(lltype.Bool) op = SpaceOperation('direct_call', [const(func), v1, v2], v3) - tr = Transformer(FakeCPU(), FakeBuiltinCallControl()) + cc = FakeBuiltinCallControl() + tr = Transformer(FakeCPU(), cc) op1 = tr.rewrite_operation(op) assert op1.opname == 'residual_call_r_i' assert op1.args[0].value == func @@ -901,9 +917,9 @@ def test_unicode_eq_checknull_char(): assert op1.args[2] == ListOfKind('ref', [v1, v2]) assert op1.result == v3 # test that the OS_UNIEQ_* functions are registered - cifo = effectinfo._callinfo_for_oopspec - assert effectinfo.EffectInfo.OS_UNIEQ_SLICE_NONNULL in cifo - assert effectinfo.EffectInfo.OS_UNIEQ_CHECKNULL_CHAR in cifo + cic = cc.callinfocollection + assert cic.has_oopspec(effectinfo.EffectInfo.OS_UNIEQ_SLICE_NONNULL) + assert cic.has_oopspec(effectinfo.EffectInfo.OS_UNIEQ_CHECKNULL_CHAR) def test_list_ll_arraycopy(): from pypy.rlib.rgc import ll_arraycopy diff --git a/pypy/jit/metainterp/blackhole.py b/pypy/jit/metainterp/blackhole.py index 700280971b..c67dc8b2fa 100644 --- a/pypy/jit/metainterp/blackhole.py +++ b/pypy/jit/metainterp/blackhole.py @@ -774,6 +774,10 @@ class BlackholeInterpreter(object): def bhimpl_float_assert_green(x): pass + @arguments(returns="i") + def bhimpl_current_trace_length(): + return -1 + # ---------- # the main hints and recursive calls diff --git a/pypy/jit/metainterp/compile.py b/pypy/jit/metainterp/compile.py index 31df49b3ff..342898daab 100644 --- a/pypy/jit/metainterp/compile.py +++ b/pypy/jit/metainterp/compile.py @@ -1,4 +1,5 @@ +from pypy.rpython.lltypesystem import lltype from pypy.rpython.ootypesystem import ootype from pypy.objspace.flow.model import Constant, Variable from pypy.rlib.objectmodel import we_are_translated @@ -13,6 +14,7 @@ from pypy.jit.metainterp.history import BoxPtr, BoxObj, BoxFloat, Const from pypy.jit.metainterp import history from pypy.jit.metainterp.typesystem import llhelper, oohelper from pypy.jit.metainterp.optimizeutil import InvalidLoop +from pypy.jit.metainterp.resume import NUMBERING from pypy.jit.codewriter import heaptracker def giveup(): @@ -52,7 +54,6 @@ def compile_new_loop(metainterp, old_loop_tokens, greenkey, start): """ history = metainterp.history loop = create_empty_loop(metainterp) - loop.greenkey = greenkey loop.inputargs = history.inputargs for box in loop.inputargs: assert isinstance(box, Box) @@ -66,7 +67,6 @@ def compile_new_loop(metainterp, old_loop_tokens, greenkey, start): loop.operations[-1].setdescr(loop_token) # patch the target of the JUMP loop.preamble = create_empty_loop(metainterp, 'Preamble ') - loop.preamble.greenkey = greenkey loop.preamble.inputargs = loop.inputargs loop.preamble.token = make_loop_token(len(loop.inputargs), jitdriver_sd) @@ -214,8 +214,7 @@ def make_done_loop_tokens(): } class ResumeDescr(AbstractFailDescr): - def __init__(self, original_greenkey): - self.original_greenkey = original_greenkey + pass class ResumeGuardDescr(ResumeDescr): _counter = 0 # if < 0, there is one counter per value; @@ -224,7 +223,7 @@ class ResumeGuardDescr(ResumeDescr): # this class also gets the following attributes stored by resume.py code rd_snapshot = None rd_frame_info_list = None - rd_numb = None + rd_numb = lltype.nullptr(NUMBERING) rd_consts = None rd_virtuals = None rd_pendingfields = None @@ -234,8 +233,7 @@ class ResumeGuardDescr(ResumeDescr): CNT_FLOAT = -0x60000000 CNT_MASK = 0x1FFFFFFF - def __init__(self, metainterp_sd, original_greenkey): - ResumeDescr.__init__(self, original_greenkey) + def __init__(self, metainterp_sd): self.metainterp_sd = metainterp_sd def store_final_boxes(self, guard_op, boxes): @@ -339,14 +337,14 @@ class ResumeGuardDescr(ResumeDescr): res.rd_pendingfields = self.rd_pendingfields def _clone_if_mutable(self): - res = ResumeGuardDescr(self.metainterp_sd, self.original_greenkey) + res = ResumeGuardDescr(self.metainterp_sd) self.copy_all_attrbutes_into(res) return res class ResumeGuardForcedDescr(ResumeGuardDescr): - def __init__(self, metainterp_sd, original_greenkey, jitdriver_sd): - ResumeGuardDescr.__init__(self, metainterp_sd, original_greenkey) + def __init__(self, metainterp_sd, jitdriver_sd): + ResumeGuardDescr.__init__(self, metainterp_sd) self.jitdriver_sd = jitdriver_sd def handle_fail(self, metainterp_sd, jitdriver_sd): @@ -413,7 +411,6 @@ class ResumeGuardForcedDescr(ResumeGuardDescr): def _clone_if_mutable(self): res = ResumeGuardForcedDescr(self.metainterp_sd, - self.original_greenkey, self.jitdriver_sd) self.copy_all_attrbutes_into(res) return res @@ -480,9 +477,8 @@ class ResumeGuardCountersFloat(AbstractResumeGuardCounters): class ResumeFromInterpDescr(ResumeDescr): - def __init__(self, original_greenkey, redkey): - ResumeDescr.__init__(self, original_greenkey) - self.redkey = redkey + def __init__(self, original_greenkey): + self.original_greenkey = original_greenkey def compile_and_attach(self, metainterp, new_loop): # We managed to create a bridge going from the interpreter @@ -491,10 +487,8 @@ class ResumeFromInterpDescr(ResumeDescr): # with completely unoptimized arguments, as in the interpreter. metainterp_sd = metainterp.staticdata jitdriver_sd = metainterp.jitdriver_sd - metainterp.history.inputargs = self.redkey - new_loop_token = make_loop_token(len(self.redkey), jitdriver_sd) - new_loop.greenkey = self.original_greenkey - new_loop.inputargs = self.redkey + redargs = new_loop.inputargs + new_loop_token = make_loop_token(len(redargs), jitdriver_sd) new_loop.token = new_loop_token send_loop_to_backend(metainterp_sd, new_loop, "entry bridge") # send the new_loop to warmspot.py, to be called directly the next time diff --git a/pypy/jit/metainterp/logger.py b/pypy/jit/metainterp/logger.py index 189462f3b5..898e1b4ff8 100644 --- a/pypy/jit/metainterp/logger.py +++ b/pypy/jit/metainterp/logger.py @@ -81,7 +81,8 @@ class Logger(object): op = operations[i] if op.getopnum() == rop.DEBUG_MERGE_POINT: loc = op.getarg(0)._get_str() - debug_print("debug_merge_point('%s')" % (loc,)) + reclev = op.getarg(1).getint() + debug_print("debug_merge_point('%s', %s)" % (loc, reclev)) continue args = ", ".join([self.repr_of_arg(memo, op.getarg(i)) for i in range(op.numargs())]) if op.result is not None: diff --git a/pypy/jit/metainterp/optimizeopt/optimizer.py b/pypy/jit/metainterp/optimizeopt/optimizer.py index 2bf3551380..cbceaf4b42 100644 --- a/pypy/jit/metainterp/optimizeopt/optimizer.py +++ b/pypy/jit/metainterp/optimizeopt/optimizer.py @@ -86,10 +86,10 @@ class OptValue(object): assert isinstance(constbox, Const) self.box = constbox self.level = LEVEL_CONSTANT - try: - val = self.box.getint() + if isinstance(constbox, ConstInt): + val = constbox.getint() self.intbound = IntBound(val, val) - except NotImplementedError: + else: self.intbound = IntUnbounded() def get_constant_class(self, cpu): diff --git a/pypy/jit/metainterp/optimizeopt/string.py b/pypy/jit/metainterp/optimizeopt/string.py index 949d0b63b9..6680885490 100644 --- a/pypy/jit/metainterp/optimizeopt/string.py +++ b/pypy/jit/metainterp/optimizeopt/string.py @@ -9,7 +9,7 @@ from pypy.jit.metainterp.optimizeopt import optimizer, virtualize from pypy.jit.metainterp.optimizeopt.optimizer import CONST_0, CONST_1 from pypy.jit.metainterp.optimizeopt.optimizer import llhelper from pypy.jit.metainterp.optimizeutil import _findall -from pypy.jit.codewriter.effectinfo import EffectInfo, callinfo_for_oopspec +from pypy.jit.codewriter.effectinfo import EffectInfo from pypy.jit.codewriter import heaptracker from pypy.rlib.unroll import unrolling_iterable from pypy.rlib.objectmodel import specialize, we_are_translated @@ -642,7 +642,8 @@ class OptString(optimizer.Optimization): def generate_modified_call(self, oopspecindex, args, result, mode): oopspecindex += mode.OS_offset - calldescr, func = callinfo_for_oopspec(oopspecindex) + cic = self.optimizer.metainterp_sd.callinfocollection + calldescr, func = cic.callinfo_for_oopspec(oopspecindex) op = ResOperation(rop.CALL, [ConstInt(func)] + args, result, descr=calldescr) self.optimizer.newoperations.append(op) diff --git a/pypy/jit/metainterp/pyjitpl.py b/pypy/jit/metainterp/pyjitpl.py index 35b8fc0360..abb8fc84d3 100644 --- a/pypy/jit/metainterp/pyjitpl.py +++ b/pypy/jit/metainterp/pyjitpl.py @@ -14,7 +14,7 @@ from pypy.jit.metainterp import executor from pypy.jit.metainterp.logger import Logger from pypy.jit.metainterp.jitprof import EmptyProfiler from pypy.jit.metainterp.jitprof import GUARDS, RECORDED_OPS, ABORT_ESCAPE -from pypy.jit.metainterp.jitprof import ABORT_TOO_LONG, ABORT_BRIDGE +from pypy.jit.metainterp.jitprof import ABORT_TOO_LONG from pypy.jit.metainterp.jitexc import JitException, get_llexception from pypy.rlib.rarithmetic import intmask from pypy.rlib.objectmodel import specialize @@ -820,7 +820,8 @@ class MIFrame(object): jitdriver_sd = self.metainterp.staticdata.jitdrivers_sd[jdindex] self.verify_green_args(jitdriver_sd, greenboxes) # xxx we may disable the following line in some context later - self.debug_merge_point(jitdriver_sd, greenboxes) + self.debug_merge_point(jitdriver_sd, self.metainterp.in_recursion, + greenboxes) if self.metainterp.seen_loop_header_for_jdindex < 0: if not jitdriver_sd.no_loop_header or not any_operation: return @@ -860,13 +861,13 @@ class MIFrame(object): assembler_call=True) raise ChangeFrame - def debug_merge_point(self, jitdriver_sd, greenkey): + def debug_merge_point(self, jitdriver_sd, in_recursion, greenkey): # debugging: produce a DEBUG_MERGE_POINT operation loc = jitdriver_sd.warmstate.get_location_str(greenkey) debug_print(loc) constloc = self.metainterp.cpu.ts.conststr(loc) self.metainterp.history.record(rop.DEBUG_MERGE_POINT, - [constloc], None) + [constloc, ConstInt(in_recursion)], None) @arguments("box", "label") def opimpl_goto_if_exception_mismatch(self, vtablebox, next_exc_target): @@ -948,6 +949,11 @@ class MIFrame(object): opimpl_ref_assert_green = _opimpl_assert_green opimpl_float_assert_green = _opimpl_assert_green + @arguments() + def opimpl_current_trace_length(self): + trace_length = len(self.metainterp.history.operations) + return ConstInt(trace_length) + @arguments("box") def opimpl_virtual_ref(self, box): # Details on the content of metainterp.virtualref_boxes: @@ -1042,14 +1048,11 @@ class MIFrame(object): else: moreargs = list(extraargs) metainterp_sd = metainterp.staticdata - original_greenkey = metainterp.resumekey.original_greenkey if opnum == rop.GUARD_NOT_FORCED: resumedescr = compile.ResumeGuardForcedDescr(metainterp_sd, - original_greenkey, metainterp.jitdriver_sd) else: - resumedescr = compile.ResumeGuardDescr(metainterp_sd, - original_greenkey) + resumedescr = compile.ResumeGuardDescr(metainterp_sd) guard_op = metainterp.history.record(opnum, moreargs, None, descr=resumedescr) virtualizable_boxes = None @@ -1255,6 +1258,7 @@ class MetaInterpStaticData(object): # self.jitdrivers_sd = codewriter.callcontrol.jitdrivers_sd self.virtualref_info = codewriter.callcontrol.virtualref_info + self.callinfocollection = codewriter.callcontrol.callinfocollection self.setup_jitdrivers_sd(optimizer) # # store this information for fastpath of call_assembler @@ -1618,7 +1622,7 @@ class MetaInterp(object): assert jitdriver_sd is self.jitdriver_sd self.create_empty_history() try: - original_boxes = self.initialize_original_boxes(jitdriver_sd,*args) + original_boxes = self.initialize_original_boxes(jitdriver_sd, *args) return self._compile_and_run_once(original_boxes) finally: self.staticdata.profiler.end_tracing() @@ -1629,9 +1633,8 @@ class MetaInterp(object): self.current_merge_points = [(original_boxes, 0)] num_green_args = self.jitdriver_sd.num_green_args original_greenkey = original_boxes[:num_green_args] - redkey = original_boxes[num_green_args:] - self.resumekey = compile.ResumeFromInterpDescr(original_greenkey, - redkey) + self.resumekey = compile.ResumeFromInterpDescr(original_greenkey) + self.history.inputargs = original_boxes[num_green_args:] self.seen_loop_header_for_jdindex = -1 try: self.interpret() @@ -1653,11 +1656,7 @@ class MetaInterp(object): debug_stop('jit-tracing') def _handle_guard_failure(self, key): - original_greenkey = key.original_greenkey - # notice that here we just put the greenkey - # use -1 to mark that we will have to give up - # because we cannot reconstruct the beginning of the proper loop - self.current_merge_points = [(original_greenkey, -1)] + self.current_merge_points = [] self.resumekey = key self.seen_loop_header_for_jdindex = -1 try: @@ -1721,7 +1720,7 @@ class MetaInterp(object): num_green_args = self.jitdriver_sd.num_green_args for j in range(len(self.current_merge_points)-1, -1, -1): original_boxes, start = self.current_merge_points[j] - assert len(original_boxes) == len(live_arg_boxes) or start < 0 + assert len(original_boxes) == len(live_arg_boxes) for i in range(num_green_args): box1 = original_boxes[i] box2 = live_arg_boxes[i] @@ -1730,10 +1729,6 @@ class MetaInterp(object): break else: # Found! Compile it as a loop. - if start < 0: - # we cannot reconstruct the beginning of the proper loop - raise SwitchToBlackhole(ABORT_BRIDGE) - # raises in case it works -- which is the common case self.compile(original_boxes, live_arg_boxes, start) # creation of the loop was cancelled! diff --git a/pypy/jit/metainterp/resoperation.py b/pypy/jit/metainterp/resoperation.py index eea9079a9a..08e2e7126c 100644 --- a/pypy/jit/metainterp/resoperation.py +++ b/pypy/jit/metainterp/resoperation.py @@ -143,6 +143,16 @@ class AbstractResOp(object): def can_raise(self): return rop._CANRAISE_FIRST <= self.getopnum() <= rop._CANRAISE_LAST + def is_malloc(self): + # a slightly different meaning from can_malloc + return rop._MALLOC_FIRST <= self.getopnum() <= rop._MALLOC_LAST + + def can_malloc(self): + return self.is_call() or self.is_malloc() + + def is_call(self): + return rop._CALL_FIRST <= self.getopnum() <= rop._CALL_LAST + def is_ovf(self): return rop._OVF_FIRST <= self.getopnum() <= rop._OVF_LAST @@ -441,9 +451,13 @@ _oplist = [ 'GETARRAYITEM_RAW/2d', 'GETFIELD_GC/1d', 'GETFIELD_RAW/1d', + '_MALLOC_FIRST', 'NEW/0d', 'NEW_WITH_VTABLE/1', 'NEW_ARRAY/1d', + 'NEWSTR/1', + 'NEWUNICODE/1', + '_MALLOC_LAST', 'FORCE_TOKEN/0', 'VIRTUAL_REF/2', # removed before it's passed to the backend '_NOSIDEEFFECT_LAST', # ----- end of no_side_effect operations ----- @@ -452,19 +466,18 @@ _oplist = [ 'SETARRAYITEM_RAW/3d', 'SETFIELD_GC/2d', 'SETFIELD_RAW/2d', - 'NEWSTR/1', 'STRSETITEM/3', 'UNICODESETITEM/3', - 'NEWUNICODE/1', #'RUNTIMENEW/1', # ootype operation 'COND_CALL_GC_WB/2d', # [objptr, newvalue] (for the write barrier) - 'DEBUG_MERGE_POINT/1', # debugging only + 'DEBUG_MERGE_POINT/2', # debugging only 'JIT_DEBUG/*', # debugging only 'VIRTUAL_REF_FINISH/2', # removed before it's passed to the backend 'COPYSTRCONTENT/5', # src, dst, srcstart, dststart, length 'COPYUNICODECONTENT/5', '_CANRAISE_FIRST', # ----- start of can_raise operations ----- + '_CALL_FIRST', 'CALL/*d', 'CALL_ASSEMBLER/*d', # call already compiled assembler 'CALL_MAY_FORCE/*d', @@ -473,6 +486,7 @@ _oplist = [ #'OOSEND_PURE', # ootype operation 'CALL_PURE/*d', # removed before it's passed to the backend # CALL_PURE(result, func, arg_1,..,arg_n) + '_CALL_LAST', '_CANRAISE_LAST', # ----- end of can_raise operations ----- '_OVF_FIRST', # ----- start of is_ovf operations ----- diff --git a/pypy/jit/metainterp/resume.py b/pypy/jit/metainterp/resume.py index 338bba1065..89a5d28354 100644 --- a/pypy/jit/metainterp/resume.py +++ b/pypy/jit/metainterp/resume.py @@ -4,8 +4,7 @@ from pypy.jit.metainterp.history import BoxInt, BoxPtr, BoxFloat from pypy.jit.metainterp.history import INT, REF, FLOAT, HOLE from pypy.jit.metainterp.resoperation import rop from pypy.jit.metainterp import jitprof -from pypy.jit.codewriter.effectinfo import EffectInfo, callinfo_for_oopspec -from pypy.jit.codewriter.effectinfo import funcptr_for_oopspec +from pypy.jit.codewriter.effectinfo import EffectInfo from pypy.rpython.lltypesystem import lltype, llmemory, rffi, rstr from pypy.rlib import rarithmetic from pypy.rlib.objectmodel import we_are_translated, specialize @@ -66,12 +65,21 @@ def capture_resumedata(framestack, virtualizable_boxes, virtualref_boxes, snapshot = Snapshot(snapshot, boxes) storage.rd_snapshot = snapshot -class Numbering(object): - __slots__ = ('prev', 'nums') - - def __init__(self, prev, nums): - self.prev = prev - self.nums = nums +# +# The following is equivalent to the RPython-level declaration: +# +# class Numbering: __slots__ = ['prev', 'nums'] +# +# except that it is more compact in translated programs, because the +# array 'nums' is inlined in the single NUMBERING object. This is +# important because this is often the biggest single consumer of memory +# in a pypy-c-jit. +# +NUMBERINGP = lltype.Ptr(lltype.GcForwardReference()) +NUMBERING = lltype.GcStruct('Numbering', + ('prev', NUMBERINGP), + ('nums', lltype.Array(rffi.SHORT))) +NUMBERINGP.TO.become(NUMBERING) TAGMASK = 3 @@ -163,7 +171,7 @@ class ResumeDataLoopMemo(object): def number(self, values, snapshot): if snapshot is None: - return None, {}, 0 + return lltype.nullptr(NUMBERING), {}, 0 if snapshot in self.numberings: numb, liveboxes, v = self.numberings[snapshot] return numb, liveboxes.copy(), v @@ -172,7 +180,7 @@ class ResumeDataLoopMemo(object): n = len(liveboxes)-v boxes = snapshot.boxes length = len(boxes) - nums = [UNASSIGNED] * length + numb = lltype.malloc(NUMBERING, length) for i in range(length): box = boxes[i] value = values.get(box, None) @@ -191,9 +199,9 @@ class ResumeDataLoopMemo(object): tagged = tag(n, TAGBOX) n += 1 liveboxes[box] = tagged - nums[i] = tagged + numb.nums[i] = tagged # - numb = Numbering(numb1, nums) + numb.prev = numb1 self.numberings[snapshot] = numb, liveboxes, v return numb, liveboxes.copy(), v @@ -298,7 +306,7 @@ class ResumeDataVirtualAdder(object): # compute the numbering storage = self.storage # make sure that nobody attached resume data to this guard yet - assert storage.rd_numb is None + assert not storage.rd_numb snapshot = storage.rd_snapshot assert snapshot is not None # is that true? numb, liveboxes_from_env, v = self.memo.number(values, snapshot) @@ -724,34 +732,36 @@ class ResumeDataBoxReader(AbstractResumeDataReader): self.boxes_f = boxes_f self._prepare_next_section(info) - def consume_virtualizable_boxes(self, vinfo, nums): + def consume_virtualizable_boxes(self, vinfo, numb): # we have to ignore the initial part of 'nums' (containing vrefs), # find the virtualizable from nums[-1], and use it to know how many # boxes of which type we have to return. This does not write # anything into the virtualizable. - virtualizablebox = self.decode_ref(nums[-1]) + index = len(numb.nums) - 1 + virtualizablebox = self.decode_ref(numb.nums[index]) virtualizable = vinfo.unwrap_virtualizable_box(virtualizablebox) - return vinfo.load_list_of_boxes(virtualizable, self, nums) + return vinfo.load_list_of_boxes(virtualizable, self, numb) - def consume_virtualref_boxes(self, nums, end): + def consume_virtualref_boxes(self, numb, end): # Returns a list of boxes, assumed to be all BoxPtrs. # We leave up to the caller to call vrefinfo.continue_tracing(). assert (end & 1) == 0 - return [self.decode_ref(nums[i]) for i in range(end)] + return [self.decode_ref(numb.nums[i]) for i in range(end)] def consume_vref_and_vable_boxes(self, vinfo, ginfo): - nums = self.cur_numb.nums - self.cur_numb = self.cur_numb.prev + numb = self.cur_numb + self.cur_numb = numb.prev if vinfo is not None: - virtualizable_boxes = self.consume_virtualizable_boxes(vinfo, nums) - end = len(nums) - len(virtualizable_boxes) + virtualizable_boxes = self.consume_virtualizable_boxes(vinfo, numb) + end = len(numb.nums) - len(virtualizable_boxes) elif ginfo is not None: - virtualizable_boxes = [self.decode_ref(nums[-1])] - end = len(nums) - 1 + index = len(numb.nums) - 1 + virtualizable_boxes = [self.decode_ref(numb.nums[index])] + end = len(numb.nums) - 1 else: virtualizable_boxes = None - end = len(nums) - virtualref_boxes = self.consume_virtualref_boxes(nums, end) + end = len(numb.nums) + virtualref_boxes = self.consume_virtualref_boxes(numb, end) return virtualizable_boxes, virtualref_boxes def allocate_with_vtable(self, known_class): @@ -775,14 +785,16 @@ class ResumeDataBoxReader(AbstractResumeDataReader): strbox, ConstInt(index), charbox) def concat_strings(self, str1num, str2num): - calldescr, func = callinfo_for_oopspec(EffectInfo.OS_STR_CONCAT) + cic = self.metainterp.staticdata.callinfocollection + calldescr, func = cic.callinfo_for_oopspec(EffectInfo.OS_STR_CONCAT) str1box = self.decode_box(str1num, REF) str2box = self.decode_box(str2num, REF) return self.metainterp.execute_and_record_varargs( rop.CALL, [ConstInt(func), str1box, str2box], calldescr) def slice_string(self, strnum, startnum, lengthnum): - calldescr, func = callinfo_for_oopspec(EffectInfo.OS_STR_SLICE) + cic = self.metainterp.staticdata.callinfocollection + calldescr, func = cic.callinfo_for_oopspec(EffectInfo.OS_STR_SLICE) strbox = self.decode_box(strnum, REF) startbox = self.decode_box(startnum, INT) lengthbox = self.decode_box(lengthnum, INT) @@ -801,14 +813,16 @@ class ResumeDataBoxReader(AbstractResumeDataReader): strbox, ConstInt(index), charbox) def concat_unicodes(self, str1num, str2num): - calldescr, func = callinfo_for_oopspec(EffectInfo.OS_UNI_CONCAT) + cic = self.metainterp.staticdata.callinfocollection + calldescr, func = cic.callinfo_for_oopspec(EffectInfo.OS_UNI_CONCAT) str1box = self.decode_box(str1num, REF) str2box = self.decode_box(str2num, REF) return self.metainterp.execute_and_record_varargs( rop.CALL, [ConstInt(func), str1box, str2box], calldescr) def slice_unicode(self, strnum, startnum, lengthnum): - calldescr, func = callinfo_for_oopspec(EffectInfo.OS_UNI_SLICE) + cic = self.metainterp.staticdata.callinfocollection + calldescr, func = cic.callinfo_for_oopspec(EffectInfo.OS_UNI_SLICE) strbox = self.decode_box(strnum, REF) startbox = self.decode_box(startnum, INT) lengthbox = self.decode_box(lengthnum, INT) @@ -904,8 +918,8 @@ class ResumeDataBoxReader(AbstractResumeDataReader): def blackhole_from_resumedata(blackholeinterpbuilder, jitdriver_sd, storage, all_virtuals=None): - resumereader = ResumeDataDirectReader(blackholeinterpbuilder.cpu, storage, - all_virtuals) + resumereader = ResumeDataDirectReader(blackholeinterpbuilder.metainterp_sd, + storage, all_virtuals) vinfo = jitdriver_sd.virtualizable_info ginfo = jitdriver_sd.greenfield_info vrefinfo = blackholeinterpbuilder.metainterp_sd.virtualref_info @@ -940,7 +954,7 @@ def blackhole_from_resumedata(blackholeinterpbuilder, jitdriver_sd, storage, return firstbh def force_from_resumedata(metainterp_sd, storage, vinfo, ginfo): - resumereader = ResumeDataDirectReader(metainterp_sd.cpu, storage) + resumereader = ResumeDataDirectReader(metainterp_sd, storage) resumereader.handling_async_forcing() vrefinfo = metainterp_sd.virtualref_info resumereader.consume_vref_and_vable(vrefinfo, vinfo, ginfo) @@ -954,8 +968,9 @@ class ResumeDataDirectReader(AbstractResumeDataReader): # 1: in handle_async_forcing # 2: resuming from the GUARD_NOT_FORCED - def __init__(self, cpu, storage, all_virtuals=None): - self._init(cpu, storage) + def __init__(self, metainterp_sd, storage, all_virtuals=None): + self._init(metainterp_sd.cpu, storage) + self.callinfocollection = metainterp_sd.callinfocollection if all_virtuals is None: # common case self._prepare(storage) else: @@ -974,23 +989,24 @@ class ResumeDataDirectReader(AbstractResumeDataReader): info = blackholeinterp.get_current_position_info() self._prepare_next_section(info) - def consume_virtualref_info(self, vrefinfo, nums, end): + def consume_virtualref_info(self, vrefinfo, numb, end): # we have to decode a list of references containing pairs # [..., virtual, vref, ...] stopping at 'end' assert (end & 1) == 0 for i in range(0, end, 2): - virtual = self.decode_ref(nums[i]) - vref = self.decode_ref(nums[i+1]) + virtual = self.decode_ref(numb.nums[i]) + vref = self.decode_ref(numb.nums[i+1]) # For each pair, we store the virtual inside the vref. vrefinfo.continue_tracing(vref, virtual) - def consume_vable_info(self, vinfo, nums): + def consume_vable_info(self, vinfo, numb): # we have to ignore the initial part of 'nums' (containing vrefs), # find the virtualizable from nums[-1], load all other values # from the CPU stack, and copy them into the virtualizable if vinfo is None: - return len(nums) - virtualizable = self.decode_ref(nums[-1]) + return len(numb.nums) + index = len(numb.nums) - 1 + virtualizable = self.decode_ref(numb.nums[index]) virtualizable = vinfo.cast_gcref_to_vtype(virtualizable) if self.resume_after_guard_not_forced == 1: # in the middle of handle_async_forcing() @@ -1002,7 +1018,7 @@ class ResumeDataDirectReader(AbstractResumeDataReader): # is and stays 0. Note the call to reset_vable_token() in # warmstate.py. assert not virtualizable.vable_token - return vinfo.write_from_resume_data_partial(virtualizable, self, nums) + return vinfo.write_from_resume_data_partial(virtualizable, self, numb) def load_value_of_type(self, TYPE, tagged): from pypy.jit.metainterp.warmstate import specialize_value @@ -1019,12 +1035,12 @@ class ResumeDataDirectReader(AbstractResumeDataReader): load_value_of_type._annspecialcase_ = 'specialize:arg(1)' def consume_vref_and_vable(self, vrefinfo, vinfo, ginfo): - nums = self.cur_numb.nums - self.cur_numb = self.cur_numb.prev + numb = self.cur_numb + self.cur_numb = numb.prev if self.resume_after_guard_not_forced != 2: - end_vref = self.consume_vable_info(vinfo, nums) + end_vref = self.consume_vable_info(vinfo, numb) if ginfo is not None: end_vref -= 1 - self.consume_virtualref_info(vrefinfo, nums, end_vref) + self.consume_virtualref_info(vrefinfo, numb, end_vref) def allocate_with_vtable(self, known_class): from pypy.jit.metainterp.executor import exec_new_with_vtable @@ -1048,7 +1064,8 @@ class ResumeDataDirectReader(AbstractResumeDataReader): str2 = self.decode_ref(str2num) str1 = lltype.cast_opaque_ptr(lltype.Ptr(rstr.STR), str1) str2 = lltype.cast_opaque_ptr(lltype.Ptr(rstr.STR), str2) - funcptr = funcptr_for_oopspec(EffectInfo.OS_STR_CONCAT) + cic = self.callinfocollection + funcptr = cic.funcptr_for_oopspec(EffectInfo.OS_STR_CONCAT) result = funcptr(str1, str2) return lltype.cast_opaque_ptr(llmemory.GCREF, result) @@ -1057,7 +1074,8 @@ class ResumeDataDirectReader(AbstractResumeDataReader): start = self.decode_int(startnum) length = self.decode_int(lengthnum) str = lltype.cast_opaque_ptr(lltype.Ptr(rstr.STR), str) - funcptr = funcptr_for_oopspec(EffectInfo.OS_STR_SLICE) + cic = self.callinfocollection + funcptr = cic.funcptr_for_oopspec(EffectInfo.OS_STR_SLICE) result = funcptr(str, start, start + length) return lltype.cast_opaque_ptr(llmemory.GCREF, result) @@ -1073,7 +1091,8 @@ class ResumeDataDirectReader(AbstractResumeDataReader): str2 = self.decode_ref(str2num) str1 = lltype.cast_opaque_ptr(lltype.Ptr(rstr.UNICODE), str1) str2 = lltype.cast_opaque_ptr(lltype.Ptr(rstr.UNICODE), str2) - funcptr = funcptr_for_oopspec(EffectInfo.OS_UNI_CONCAT) + cic = self.callinfocollection + funcptr = cic.funcptr_for_oopspec(EffectInfo.OS_UNI_CONCAT) result = funcptr(str1, str2) return lltype.cast_opaque_ptr(llmemory.GCREF, result) @@ -1082,7 +1101,8 @@ class ResumeDataDirectReader(AbstractResumeDataReader): start = self.decode_int(startnum) length = self.decode_int(lengthnum) str = lltype.cast_opaque_ptr(lltype.Ptr(rstr.UNICODE), str) - funcptr = funcptr_for_oopspec(EffectInfo.OS_UNI_SLICE) + cic = self.callinfocollection + funcptr = cic.funcptr_for_oopspec(EffectInfo.OS_UNI_SLICE) result = funcptr(str, start, start + length) return lltype.cast_opaque_ptr(llmemory.GCREF, result) @@ -1173,8 +1193,9 @@ def dump_storage(storage, liveboxes): 'at', compute_unique_id(frameinfo)) frameinfo = frameinfo.prev numb = storage.rd_numb - while numb is not None: - debug_print('\tnumb', str([untag(i) for i in numb.nums]), + while numb: + debug_print('\tnumb', str([untag(numb.nums[i]) + for i in range(len(numb.nums))]), 'at', compute_unique_id(numb)) numb = numb.prev for const in storage.rd_consts: diff --git a/pypy/jit/metainterp/test/test_basic.py b/pypy/jit/metainterp/test/test_basic.py index 6b994f5d75..a96440a7bf 100644 --- a/pypy/jit/metainterp/test/test_basic.py +++ b/pypy/jit/metainterp/test/test_basic.py @@ -3,6 +3,7 @@ import sys from pypy.rlib.jit import JitDriver, we_are_jitted, hint, dont_look_inside from pypy.rlib.jit import OPTIMIZER_FULL, OPTIMIZER_SIMPLE, loop_invariant from pypy.rlib.jit import jit_debug, assert_green, AssertGreenFailed +from pypy.rlib.jit import unroll_safe, current_trace_length from pypy.jit.metainterp.warmspot import ll_meta_interp, get_stats from pypy.jit.backend.llgraph import runner from pypy.jit.metainterp import pyjitpl, history @@ -1837,6 +1838,31 @@ class BasicTests: 'int_add': 1, 'int_mul': 1, 'int_sub': 2, 'int_gt': 2, 'jump': 2}) + def test_current_trace_length(self): + myjitdriver = JitDriver(greens = ['g'], reds = ['x']) + @dont_look_inside + def residual(): + print "hi there" + @unroll_safe + def loop(g): + y = 0 + while y < g: + residual() + y += 1 + def f(x, g): + n = 0 + while x > 0: + myjitdriver.can_enter_jit(x=x, g=g) + myjitdriver.jit_merge_point(x=x, g=g) + loop(g) + x -= 1 + n = current_trace_length() + return n + res = self.meta_interp(f, [5, 8]) + assert 14 < res < 42 + res = self.meta_interp(f, [5, 2]) + assert 4 < res < 14 + class TestOOtype(BasicTests, OOJitMixin): diff --git a/pypy/jit/metainterp/test/test_compile.py b/pypy/jit/metainterp/test/test_compile.py index 98edd3f7e8..cddc61080a 100644 --- a/pypy/jit/metainterp/test/test_compile.py +++ b/pypy/jit/metainterp/test/test_compile.py @@ -85,7 +85,7 @@ def test_compile_new_loop(): metainterp.history.inputargs = loop.inputargs[:] # loop_tokens = [] - loop_token = compile_new_loop(metainterp, loop_tokens, [], 0) + loop_token = compile_new_loop(metainterp, loop_tokens, 0) assert loop_tokens == [loop_token] assert loop_token.number == 1 assert staticdata.globaldata.loopnumbering == 2 @@ -101,7 +101,7 @@ def test_compile_new_loop(): metainterp.history.operations = loop.operations[:] metainterp.history.inputargs = loop.inputargs[:] # - loop_token_2 = compile_new_loop(metainterp, loop_tokens, [], 0) + loop_token_2 = compile_new_loop(metainterp, loop_tokens, 0) assert loop_token_2 is loop_token assert loop_tokens == [loop_token] assert len(cpu.seen) == 0 diff --git a/pypy/jit/metainterp/test/test_del.py b/pypy/jit/metainterp/test/test_del.py index 8ce32f6535..5ea0dc817f 100644 --- a/pypy/jit/metainterp/test/test_del.py +++ b/pypy/jit/metainterp/test/test_del.py @@ -85,6 +85,7 @@ class TestLLtype(DelTests, LLJitMixin): def test_signal_action(self): from pypy.module.signal.interp_signal import SignalActionFlag action = SignalActionFlag() + action.has_bytecode_counter = True # myjitdriver = JitDriver(greens = [], reds = ['n', 'x']) class X: @@ -92,17 +93,17 @@ class TestLLtype(DelTests, LLJitMixin): # def f(n): x = X() - while n > 0: + action.reset_ticker(n) + while True: myjitdriver.can_enter_jit(n=n, x=x) myjitdriver.jit_merge_point(n=n, x=x) x.foo = n n -= 1 - if action.get() != 0: + if action.decrement_ticker(1) < 0: break - action.set(0) return 42 self.meta_interp(f, [20]) - self.check_loops(getfield_raw=1, call=0, call_pure=0) + self.check_loops(getfield_raw=1, setfield_raw=1, call=0, call_pure=0) class TestOOtype(DelTests, OOJitMixin): def setup_class(cls): diff --git a/pypy/jit/metainterp/test/test_logger.py b/pypy/jit/metainterp/test/test_logger.py index 24618f6a09..8a2eeaa51e 100644 --- a/pypy/jit/metainterp/test/test_logger.py +++ b/pypy/jit/metainterp/test/test_logger.py @@ -97,7 +97,7 @@ class TestLogger(object): def test_debug_merge_point(self): inp = ''' [] - debug_merge_point("info") + debug_merge_point("info", 0) ''' loop, oloop = self.reparse(inp, check_equal=False) assert loop.operations[0].getarg(0)._get_str() == 'info' diff --git a/pypy/jit/metainterp/test/test_optimizeopt.py b/pypy/jit/metainterp/test/test_optimizeopt.py index baf8e65777..74393e540d 100644 --- a/pypy/jit/metainterp/test/test_optimizeopt.py +++ b/pypy/jit/metainterp/test/test_optimizeopt.py @@ -26,6 +26,100 @@ class FakeMetaInterpStaticData(object): self.options = Fake() self.globaldata = Fake() +def test_store_final_boxes_in_guard(): + from pypy.jit.metainterp.compile import ResumeGuardDescr + from pypy.jit.metainterp.resume import tag, TAGBOX + b0 = BoxInt() + b1 = BoxInt() + opt = optimizeopt.Optimizer(FakeMetaInterpStaticData(LLtypeMixin.cpu), + None) + fdescr = ResumeGuardDescr(None) + op = ResOperation(rop.GUARD_TRUE, ['dummy'], None, descr=fdescr) + # setup rd data + fi0 = resume.FrameInfo(None, "code0", 11) + fdescr.rd_frame_info_list = resume.FrameInfo(fi0, "code1", 33) + snapshot0 = resume.Snapshot(None, [b0]) + fdescr.rd_snapshot = resume.Snapshot(snapshot0, [b1]) + # + opt.store_final_boxes_in_guard(op) + if op.getfailargs() == [b0, b1]: + assert list(fdescr.rd_numb.nums) == [tag(1, TAGBOX)] + assert list(fdescr.rd_numb.prev.nums) == [tag(0, TAGBOX)] + else: + assert op.getfailargs() == [b1, b0] + assert list(fdescr.rd_numb.nums) == [tag(0, TAGBOX)] + assert list(fdescr.rd_numb.prev.nums) == [tag(1, TAGBOX)] + assert fdescr.rd_virtuals is None + assert fdescr.rd_consts == [] + +def test_sharing_field_lists_of_virtual(): + class FakeOptimizer(object): + class cpu(object): + pass + opt = FakeOptimizer() + virt1 = virtualize.AbstractVirtualStructValue(opt, None) + lst1 = virt1._get_field_descr_list() + assert lst1 == [] + lst2 = virt1._get_field_descr_list() + assert lst1 is lst2 + virt1.setfield(LLtypeMixin.valuedescr, optimizeopt.OptValue(None)) + lst3 = virt1._get_field_descr_list() + assert lst3 == [LLtypeMixin.valuedescr] + lst4 = virt1._get_field_descr_list() + assert lst3 is lst4 + + virt2 = virtualize.AbstractVirtualStructValue(opt, None) + lst5 = virt2._get_field_descr_list() + assert lst5 is lst1 + virt2.setfield(LLtypeMixin.valuedescr, optimizeopt.OptValue(None)) + lst6 = virt1._get_field_descr_list() + assert lst6 is lst3 + +def test_reuse_vinfo(): + class FakeVInfo(object): + def set_content(self, fieldnums): + self.fieldnums = fieldnums + def equals(self, fieldnums): + return self.fieldnums == fieldnums + class FakeVirtualValue(virtualize.AbstractVirtualValue): + def _make_virtual(self, *args): + return FakeVInfo() + v1 = FakeVirtualValue(None, None, None) + vinfo1 = v1.make_virtual_info(None, [1, 2, 4]) + vinfo2 = v1.make_virtual_info(None, [1, 2, 4]) + assert vinfo1 is vinfo2 + vinfo3 = v1.make_virtual_info(None, [1, 2, 6]) + assert vinfo3 is not vinfo2 + vinfo4 = v1.make_virtual_info(None, [1, 2, 6]) + assert vinfo3 is vinfo4 + +def test_descrlist_dict(): + from pypy.jit.metainterp import optimizeutil + h1 = optimizeutil.descrlist_hash([]) + h2 = optimizeutil.descrlist_hash([LLtypeMixin.valuedescr]) + h3 = optimizeutil.descrlist_hash( + [LLtypeMixin.valuedescr, LLtypeMixin.nextdescr]) + assert h1 != h2 + assert h2 != h3 + assert optimizeutil.descrlist_eq([], []) + assert not optimizeutil.descrlist_eq([], [LLtypeMixin.valuedescr]) + assert optimizeutil.descrlist_eq([LLtypeMixin.valuedescr], + [LLtypeMixin.valuedescr]) + assert not optimizeutil.descrlist_eq([LLtypeMixin.valuedescr], + [LLtypeMixin.nextdescr]) + assert optimizeutil.descrlist_eq([LLtypeMixin.valuedescr, LLtypeMixin.nextdescr], + [LLtypeMixin.valuedescr, LLtypeMixin.nextdescr]) + assert not optimizeutil.descrlist_eq([LLtypeMixin.nextdescr, LLtypeMixin.valuedescr], + [LLtypeMixin.valuedescr, LLtypeMixin.nextdescr]) + + # descrlist_eq should compare by identity of the descrs, not by the result + # of sort_key + class FakeDescr(object): + def sort_key(self): + return 1 + + assert not optimizeutil.descrlist_eq([FakeDescr()], [FakeDescr()]) + # ____________________________________________________________ class Storage(compile.ResumeGuardDescr): "for tests." @@ -76,6 +170,8 @@ class BaseTestOptimizeOpt(BaseTest): metainterp_sd = FakeMetaInterpStaticData(self.cpu) if hasattr(self, 'vrefinfo'): metainterp_sd.virtualref_info = self.vrefinfo + if hasattr(self, 'callinfocollection'): + metainterp_sd.callinfocollection = self.callinfocollection optimize_loop_1(metainterp_sd, loop) # @@ -1430,7 +1526,7 @@ class OptimizeOptTest(BaseTestOptimizeOpt): ops = """ [p1] i1 = getfield_gc(p1, descr=valuedescr) - debug_merge_point(15) + debug_merge_point(15, 0) i2 = getfield_gc(p1, descr=valuedescr) escape(i1) escape(i2) @@ -1439,7 +1535,7 @@ class OptimizeOptTest(BaseTestOptimizeOpt): expected = """ [p1] i1 = getfield_gc(p1, descr=valuedescr) - debug_merge_point(15) + debug_merge_point(15, 0) escape(i1) escape(i1) jump(p1) @@ -4288,22 +4384,20 @@ class TestLLtype(OptimizeOptTest, LLtypeMixin): # ---------- def optimize_strunicode_loop_extradescrs(self, ops, optops, preamble=None): from pypy.jit.metainterp.optimizeopt import string - def my_callinfo_for_oopspec(oopspecindex): - calldescrtype = type(LLtypeMixin.strequaldescr) - for value in LLtypeMixin.__dict__.values(): - if isinstance(value, calldescrtype): - if (value.get_extra_info() and - value.get_extra_info().oopspecindex == oopspecindex): - # returns 0 for 'func' in this test - return value, 0 - raise AssertionError("not found: oopspecindex=%d" % oopspecindex) + class FakeCallInfoCollection: + def callinfo_for_oopspec(self, oopspecindex): + calldescrtype = type(LLtypeMixin.strequaldescr) + for value in LLtypeMixin.__dict__.values(): + if isinstance(value, calldescrtype): + extra = value.get_extra_info() + if extra and extra.oopspecindex == oopspecindex: + # returns 0 for 'func' in this test + return value, 0 + raise AssertionError("not found: oopspecindex=%d" % + oopspecindex) # - saved = string.callinfo_for_oopspec - try: - string.callinfo_for_oopspec = my_callinfo_for_oopspec - self.optimize_strunicode_loop(ops, optops, preamble) - finally: - string.callinfo_for_oopspec = saved + self.callinfocollection = FakeCallInfoCollection() + self.optimize_strunicode_loop(ops, optops, preamble) def test_str_equal_noop1(self): ops = """ diff --git a/pypy/jit/metainterp/test/test_resoperation.py b/pypy/jit/metainterp/test/test_resoperation.py index b8cebb8b7a..b64390a784 100644 --- a/pypy/jit/metainterp/test/test_resoperation.py +++ b/pypy/jit/metainterp/test/test_resoperation.py @@ -61,3 +61,10 @@ def test_instantiate(): assert op.getarglist() == ['a', 'b'] assert op.result == 'c' assert op.getdescr() is mydescr + +def test_can_malloc(): + mydescr = AbstractDescr() + assert rop.ResOperation(rop.rop.NEW, [], 'b').can_malloc() + call = rop.ResOperation(rop.rop.CALL, ['a', 'b'], 'c', descr=mydescr) + assert call.can_malloc() + assert not rop.ResOperation(rop.rop.INT_ADD, ['a', 'b'], 'c').can_malloc() diff --git a/pypy/jit/metainterp/test/test_resume.py b/pypy/jit/metainterp/test/test_resume.py index 3e0a8a8fcb..bd891d9ea6 100644 --- a/pypy/jit/metainterp/test/test_resume.py +++ b/pypy/jit/metainterp/test/test_resume.py @@ -52,6 +52,7 @@ def test_vinfo(): class MyMetaInterp: _already_allocated_resume_virtuals = None + callinfocollection = None def __init__(self, cpu=None): if cpu is None: @@ -142,6 +143,13 @@ def _next_section(reader, *expected): assert bh.written_f == expected_f +def Numbering(prev, nums): + numb = lltype.malloc(NUMBERING, len(nums)) + numb.prev = prev or lltype.nullptr(NUMBERING) + for i in range(len(nums)): + numb.nums[i] = nums[i] + return numb + def test_simple_read(): #b1, b2, b3 = [BoxInt(), BoxPtr(), BoxInt()] c1, c2, c3 = [ConstInt(111), ConstInt(222), ConstInt(333)] @@ -157,12 +165,12 @@ def test_simple_read(): storage.rd_numb = numb # cpu = MyCPU([42, gcref1, -66]) - reader = ResumeDataDirectReader(cpu, storage) + metainterp = MyMetaInterp(cpu) + reader = ResumeDataDirectReader(metainterp, storage) _next_section(reader, 42, 111, gcrefnull, 42, gcref1) _next_section(reader, 222, 333) _next_section(reader, 42, gcref1, -66) # - metainterp = MyMetaInterp(cpu) reader = ResumeDataBoxReader(storage, metainterp) bi, br, bf = [None]*3, [None]*2, [None]*0 info = MyBlackholeInterp([lltype.Signed, lltype.Signed, @@ -194,7 +202,7 @@ def test_simple_read_tagged_ints(): storage.rd_numb = numb # cpu = MyCPU([]) - reader = ResumeDataDirectReader(cpu, storage) + reader = ResumeDataDirectReader(MyMetaInterp(cpu), storage) _next_section(reader, 100) @@ -212,7 +220,7 @@ def test_prepare_virtuals(): class FakeMetainterp(object): _already_allocated_resume_virtuals = None cpu = None - reader = ResumeDataDirectReader(None, FakeStorage()) + reader = ResumeDataDirectReader(MyMetaInterp(None), FakeStorage()) assert reader.force_all_virtuals() == ["allocated", reader.virtual_default] # ____________________________________________________________ @@ -391,15 +399,15 @@ def test_FrameInfo_create(): assert fi1.pc == 3 def test_Numbering_create(): - l = [1, 2] + l = [rffi.r_short(1), rffi.r_short(2)] numb = Numbering(None, l) - assert numb.prev is None - assert numb.nums is l + assert not numb.prev + assert list(numb.nums) == l - l1 = ['b3'] + l1 = [rffi.r_short(3)] numb1 = Numbering(numb, l1) - assert numb1.prev is numb - assert numb1.nums is l1 + assert numb1.prev == numb + assert list(numb1.nums) == l1 def test_capture_resumedata(): b1, b2, b3 = [BoxInt(), BoxPtr(), BoxInt()] @@ -765,11 +773,12 @@ def test_ResumeDataLoopMemo_number(): assert liveboxes == {b1: tag(0, TAGBOX), b2: tag(1, TAGBOX), b3: tag(2, TAGBOX)} - assert numb.nums == [tag(3, TAGINT), tag(2, TAGBOX), tag(0, TAGBOX), - tag(1, TAGINT)] - assert numb.prev.nums == [tag(0, TAGBOX), tag(1, TAGINT), tag(1, TAGBOX), - tag(0, TAGBOX), tag(2, TAGINT)] - assert numb.prev.prev is None + assert list(numb.nums) == [tag(3, TAGINT), tag(2, TAGBOX), tag(0, TAGBOX), + tag(1, TAGINT)] + assert list(numb.prev.nums) == [tag(0, TAGBOX), tag(1, TAGINT), + tag(1, TAGBOX), + tag(0, TAGBOX), tag(2, TAGINT)] + assert not numb.prev.prev numb2, liveboxes2, v = memo.number({}, snap2) assert v == 0 @@ -777,9 +786,9 @@ def test_ResumeDataLoopMemo_number(): assert liveboxes2 == {b1: tag(0, TAGBOX), b2: tag(1, TAGBOX), b3: tag(2, TAGBOX)} assert liveboxes2 is not liveboxes - assert numb2.nums == [tag(3, TAGINT), tag(2, TAGBOX), tag(0, TAGBOX), - tag(3, TAGINT)] - assert numb2.prev is numb.prev + assert list(numb2.nums) == [tag(3, TAGINT), tag(2, TAGBOX), tag(0, TAGBOX), + tag(3, TAGINT)] + assert numb2.prev == numb.prev env3 = [c3, b3, b1, c3] snap3 = Snapshot(snap, env3) @@ -800,9 +809,9 @@ def test_ResumeDataLoopMemo_number(): assert v == 0 assert liveboxes3 == {b1: tag(0, TAGBOX), b2: tag(1, TAGBOX)} - assert numb3.nums == [tag(3, TAGINT), tag(4, TAGINT), tag(0, TAGBOX), - tag(3, TAGINT)] - assert numb3.prev is numb.prev + assert list(numb3.nums) == [tag(3, TAGINT), tag(4, TAGINT), tag(0, TAGBOX), + tag(3, TAGINT)] + assert numb3.prev == numb.prev # virtual env4 = [c3, b4, b1, c3] @@ -813,9 +822,9 @@ def test_ResumeDataLoopMemo_number(): assert liveboxes4 == {b1: tag(0, TAGBOX), b2: tag(1, TAGBOX), b4: tag(0, TAGVIRTUAL)} - assert numb4.nums == [tag(3, TAGINT), tag(0, TAGVIRTUAL), tag(0, TAGBOX), - tag(3, TAGINT)] - assert numb4.prev is numb.prev + assert list(numb4.nums) == [tag(3, TAGINT), tag(0, TAGVIRTUAL), + tag(0, TAGBOX), tag(3, TAGINT)] + assert numb4.prev == numb.prev env5 = [b1, b4, b5] snap5 = Snapshot(snap4, env5) @@ -826,9 +835,9 @@ def test_ResumeDataLoopMemo_number(): assert liveboxes5 == {b1: tag(0, TAGBOX), b2: tag(1, TAGBOX), b4: tag(0, TAGVIRTUAL), b5: tag(1, TAGVIRTUAL)} - assert numb5.nums == [tag(0, TAGBOX), tag(0, TAGVIRTUAL), - tag(1, TAGVIRTUAL)] - assert numb5.prev is numb4 + assert list(numb5.nums) == [tag(0, TAGBOX), tag(0, TAGVIRTUAL), + tag(1, TAGVIRTUAL)] + assert numb5.prev == numb4 def test_ResumeDataLoopMemo_number_boxes(): memo = ResumeDataLoopMemo(FakeMetaInterpStaticData()) @@ -926,7 +935,7 @@ def test_virtual_adder_int_constants(): liveboxes = modifier.finish({}) assert storage.rd_snapshot is None cpu = MyCPU([]) - reader = ResumeDataDirectReader(cpu, storage) + reader = ResumeDataDirectReader(MyMetaInterp(cpu), storage) _next_section(reader, sys.maxint, 2**16, -65) _next_section(reader, 2, 3) _next_section(reader, sys.maxint, 1, sys.maxint, 2**16) diff --git a/pypy/jit/metainterp/test/test_virtualref.py b/pypy/jit/metainterp/test/test_virtualref.py index 097f9f99c9..e0e738207f 100644 --- a/pypy/jit/metainterp/test/test_virtualref.py +++ b/pypy/jit/metainterp/test/test_virtualref.py @@ -88,7 +88,11 @@ class VRefTests: cpu.get_latest_value_int = lambda i:guard_op.getfailargs()[i].getint() cpu.get_latest_value_ref = lambda i:guard_op.getfailargs()[i].getref_base() cpu.clear_latest_values = lambda count: None - resumereader = ResumeDataDirectReader(cpu, guard_op.getdescr()) + class FakeMetaInterpSd: + callinfocollection = None + FakeMetaInterpSd.cpu = cpu + resumereader = ResumeDataDirectReader(FakeMetaInterpSd(), + guard_op.getdescr()) vrefinfo = self.metainterp.staticdata.virtualref_info lst = [] vrefinfo.continue_tracing = lambda vref, virtual: \ diff --git a/pypy/jit/metainterp/test/test_ztranslation.py b/pypy/jit/metainterp/test/test_ztranslation.py index 3987544b19..026e4d5dee 100644 --- a/pypy/jit/metainterp/test/test_ztranslation.py +++ b/pypy/jit/metainterp/test/test_ztranslation.py @@ -79,7 +79,7 @@ class TranslationTest: res = ll_meta_interp(main, [40, 5], CPUClass=self.CPUClass, type_system=self.type_system) assert res == main(40, 5) - res = rpython_ll_meta_interp(main, [40, 5], loops=2, + res = rpython_ll_meta_interp(main, [40, 5], CPUClass=self.CPUClass, type_system=self.type_system, optimizer=OPTIMIZER_FULL, @@ -120,7 +120,7 @@ class TranslationTest: res = ll_meta_interp(main, [40], CPUClass=self.CPUClass, type_system=self.type_system) assert res == main(40) - res = rpython_ll_meta_interp(main, [40], loops=2, CPUClass=self.CPUClass, + res = rpython_ll_meta_interp(main, [40], CPUClass=self.CPUClass, type_system=self.type_system, optimizer=OPTIMIZER_FULL, ProfilerClass=Profiler) diff --git a/pypy/jit/metainterp/virtualizable.py b/pypy/jit/metainterp/virtualizable.py index c6071dab9f..baa54fdb3d 100644 --- a/pypy/jit/metainterp/virtualizable.py +++ b/pypy/jit/metainterp/virtualizable.py @@ -100,48 +100,48 @@ class VirtualizableInfo: i = i + 1 assert len(boxes) == i + 1 # - def write_from_resume_data_partial(virtualizable, reader, nums): + def write_from_resume_data_partial(virtualizable, reader, numb): # Load values from the reader (see resume.py) described by # the list of numbers 'nums', and write them in their proper # place in the 'virtualizable'. This works from the end of # the list and returns the index in 'nums' of the start of # the virtualizable data found, allowing the caller to do # further processing with the start of the list. - i = len(nums) - 1 + i = len(numb.nums) - 1 assert i >= 0 for ARRAYITEMTYPE, fieldname in unroll_array_fields_rev: lst = getattr(virtualizable, fieldname) for j in range(getlength(lst)-1, -1, -1): i -= 1 assert i >= 0 - x = reader.load_value_of_type(ARRAYITEMTYPE, nums[i]) + x = reader.load_value_of_type(ARRAYITEMTYPE, numb.nums[i]) setarrayitem(lst, j, x) for FIELDTYPE, fieldname in unroll_static_fields_rev: i -= 1 assert i >= 0 - x = reader.load_value_of_type(FIELDTYPE, nums[i]) + x = reader.load_value_of_type(FIELDTYPE, numb.nums[i]) setattr(virtualizable, fieldname, x) return i # - def load_list_of_boxes(virtualizable, reader, nums): + def load_list_of_boxes(virtualizable, reader, numb): # Uses 'virtualizable' only to know the length of the arrays; # does not write anything into it. The returned list is in # the format expected of virtualizable_boxes, so it ends in # the virtualizable itself. - i = len(nums) - 1 + i = len(numb.nums) - 1 assert i >= 0 - boxes = [reader.decode_box_of_type(self.VTYPEPTR, nums[i])] + boxes = [reader.decode_box_of_type(self.VTYPEPTR, numb.nums[i])] for ARRAYITEMTYPE, fieldname in unroll_array_fields_rev: lst = getattr(virtualizable, fieldname) for j in range(getlength(lst)-1, -1, -1): i -= 1 assert i >= 0 - box = reader.decode_box_of_type(ARRAYITEMTYPE, nums[i]) + box = reader.decode_box_of_type(ARRAYITEMTYPE,numb.nums[i]) boxes.append(box) for FIELDTYPE, fieldname in unroll_static_fields_rev: i -= 1 assert i >= 0 - box = reader.decode_box_of_type(FIELDTYPE, nums[i]) + box = reader.decode_box_of_type(FIELDTYPE, numb.nums[i]) boxes.append(box) boxes.reverse() return boxes diff --git a/pypy/jit/metainterp/warmspot.py b/pypy/jit/metainterp/warmspot.py index 819fa79957..a5d36ee7b8 100644 --- a/pypy/jit/metainterp/warmspot.py +++ b/pypy/jit/metainterp/warmspot.py @@ -98,8 +98,7 @@ def jittify_and_run(interp, graph, args, repeat=1, repeat -= 1 return res -def rpython_ll_meta_interp(function, args, backendopt=True, - loops='not used right now', **kwds): +def rpython_ll_meta_interp(function, args, backendopt=True, **kwds): return ll_meta_interp(function, args, backendopt=backendopt, translate_support_code=True, **kwds) diff --git a/pypy/jit/tool/loopcounter.py b/pypy/jit/tool/loopcounter.py new file mode 100644 index 0000000000..978056f797 --- /dev/null +++ b/pypy/jit/tool/loopcounter.py @@ -0,0 +1,44 @@ +#!/usr/bin/env python +""" +Parse and display the traces produced by pypy-c-jit when PYPYLOG is set. +""" + +import autopath +import py +import sys +import optparse + +def get_timestamp(line): + import re + match = re.match(r'\[([0-9a-f]*)\] .*', line) + return int(match.group(1), 16) + +def main(logfile, options): + log = open(logfile) + loops = 0 + bridges = 0 + time0 = None + print 'timestamp,total,loops,bridges' + for line in log: + if time0 is None and line.startswith('['): + time0 = get_timestamp(line) + if '{jit-log-opt-' in line: + time_now = get_timestamp(line) + if '{jit-log-opt-loop' in line: + loops += 1 + elif '{jit-log-opt-bridge' in line: + bridges += 1 + else: + assert False, 'unknown category %s' % line + total = loops+bridges + timestamp = time_now - time0 + print '%d,%d,%d,%d' % (timestamp, total, loops, bridges) + +if __name__ == '__main__': + parser = optparse.OptionParser(usage="%prog loopfile [options]") + options, args = parser.parse_args() + if len(args) != 1: + parser.print_help() + sys.exit(2) + + main(args[0], options) diff --git a/pypy/jit/tool/oparser.py b/pypy/jit/tool/oparser.py index 6d568bb195..9ac74a1617 100644 --- a/pypy/jit/tool/oparser.py +++ b/pypy/jit/tool/oparser.py @@ -192,7 +192,7 @@ class OpParser(object): descr = None if argspec.strip(): if opname == 'debug_merge_point': - allargs = [argspec] + allargs = argspec.rsplit(', ', 1) else: allargs = [arg for arg in argspec.split(",") if arg != ''] diff --git a/pypy/jit/tool/pypytrace-mode.el b/pypy/jit/tool/pypytrace-mode.el index 77207b6d2c..6136a8331d 100644 --- a/pypy/jit/tool/pypytrace-mode.el +++ b/pypy/jit/tool/pypytrace-mode.el @@ -26,7 +26,7 @@ ("<.*FieldDescr \\([^ ]*\\)" (1 'font-lock-variable-name-face)) ;; comment out debug_merge_point, but then highlight specific part of it ("^debug_merge_point.*" . font-lock-comment-face) - ("^\\(debug_merge_point\\).*code object\\(.*\\), file \\('.*'\\), \\(line .*\\)> \\(.*\\)')" + ("^\\(debug_merge_point\\).*code object\\(.*\\), file \\('.*'\\), \\(line .*\\)> \\(.*\\)" (1 'compilation-warning t) (2 'escape-glyph t) (3 'font-lock-string-face t) diff --git a/pypy/jit/tool/pypytrace.vim b/pypy/jit/tool/pypytrace.vim index be299495e9..42f9cd3994 100644 --- a/pypy/jit/tool/pypytrace.vim +++ b/pypy/jit/tool/pypytrace.vim @@ -21,10 +21,10 @@ syn match pypyDebugMergePoint '^debug_merge_point(.\+)' hi def link pypyLoopStart Structure "hi def link pypyLoopArgs PreProc -hi def link pypyFailArgs String +hi def link pypyFailArgs Special "hi def link pypyOpName Statement -hi def link pypyDebugMergePoint Comment +hi def link pypyDebugMergePoint String hi def link pypyConstPtr Constant hi def link pypyNumber Number -hi def link pypyDescr String +hi def link pypyDescr PreProc hi def link pypyDescrField Label diff --git a/pypy/jit/metainterp/test/test_oparser.py b/pypy/jit/tool/test/test_oparser.py index ef06d4a439..a5a03314e3 100644 --- a/pypy/jit/metainterp/test/test_oparser.py +++ b/pypy/jit/tool/test/test_oparser.py @@ -141,10 +141,10 @@ def test_floats(): def test_debug_merge_point(): x = ''' [] - debug_merge_point("info") - debug_merge_point('info') - debug_merge_point('<some ('other,')> info') - debug_merge_point('(stuff) #1') + debug_merge_point("info", 0) + debug_merge_point('info', 1) + debug_merge_point('<some ('other,')> info', 1) + debug_merge_point('(stuff) #1', 1) ''' loop = parse(x) assert loop.operations[0].getarg(0)._get_str() == 'info' @@ -168,7 +168,7 @@ i4 = int_add(i0, 2) i6 = int_sub(i1, 1) i8 = int_gt(i6, 3) guard_true(i8, descr=<Guard15>) [i4, i6] -debug_merge_point('(no jitdriver.get_printable_location!)') +debug_merge_point('(no jitdriver.get_printable_location!)', 0) jump(i6, i4, descr=<Loop0>) ''' diff --git a/pypy/jit/tool/test/test_traceviewer.py b/pypy/jit/tool/test/test_traceviewer.py index b3f3a3cde1..c505c8a576 100644 --- a/pypy/jit/tool/test/test_traceviewer.py +++ b/pypy/jit/tool/test/test_traceviewer.py @@ -19,7 +19,7 @@ class TestSplitLoops(object): def test_no_of_loops(self): data = [preparse(""" # Loop 0 : loop with 39 ops - debug_merge_point('') + debug_merge_point('', 0) guard_class(p4, 141310752, descr=<Guard5>) [p0, p1] p60 = getfield_gc(p4, descr=<GcPtrFieldDescr 16>) guard_nonnull(p60, descr=<Guard6>) [p0, p1] @@ -51,7 +51,7 @@ class TestSplitLoops(object): assert loop.right.content == 'extra' def test_postparse(self): - real_loops = [FinalBlock("debug_merge_point('<code object _runCallbacks, file '/tmp/x/twisted-trunk/twisted/internet/defer.py', line 357> #40 POP_TOP')", None)] + real_loops = [FinalBlock("debug_merge_point('<code object _runCallbacks, file '/tmp/x/twisted-trunk/twisted/internet/defer.py', line 357> #40 POP_TOP', 0)", None)] postprocess(real_loops, real_loops[:], {}) assert real_loops[0].header.startswith("_runCallbacks, file '/tmp/x/twisted-trunk/twisted/internet/defer.py', line 357") diff --git a/pypy/module/__builtin__/descriptor.py b/pypy/module/__builtin__/descriptor.py index dbad647609..6c51fa3d27 100644 --- a/pypy/module/__builtin__/descriptor.py +++ b/pypy/module/__builtin__/descriptor.py @@ -96,6 +96,8 @@ class C(B): ) class W_Property(Wrappable): + _immutable_fields_ = ["w_fget", "w_fset", "w_fdel"] + def init(self, space, w_fget=None, w_fset=None, w_fdel=None, w_doc=None): self.w_fget = w_fget self.w_fset = w_fset @@ -183,4 +185,3 @@ class C(object): fget = interp_attrproperty_w('w_fget', W_Property), fset = interp_attrproperty_w('w_fset', W_Property), ) - diff --git a/pypy/module/__pypy__/__init__.py b/pypy/module/__pypy__/__init__.py index 3b1d69e2fe..c3fe0ce594 100644 --- a/pypy/module/__pypy__/__init__.py +++ b/pypy/module/__pypy__/__init__.py @@ -23,5 +23,12 @@ class Module(MixedModule): 'interp_magic.method_cache_counter') self.extra_interpdef('reset_method_cache_counter', 'interp_magic.reset_method_cache_counter') + if self.space.config.objspace.std.withmapdict: + self.extra_interpdef('mapdict_cache_counter', + 'interp_magic.mapdict_cache_counter') PYC_MAGIC = get_pyc_magic(self.space) self.extra_interpdef('PYC_MAGIC', 'space.wrap(%d)' % PYC_MAGIC) + # + from pypy.jit.backend import detect_cpu + model = detect_cpu.autodetect_main_model_and_size() + self.extra_interpdef('cpumodel', 'space.wrap(%r)' % model) diff --git a/pypy/module/__pypy__/interp_magic.py b/pypy/module/__pypy__/interp_magic.py index 71a7c89eba..0dafafe546 100644 --- a/pypy/module/__pypy__/interp_magic.py +++ b/pypy/module/__pypy__/interp_magic.py @@ -2,6 +2,7 @@ from pypy.interpreter.error import OperationError from pypy.interpreter.gateway import ObjSpace from pypy.rlib.objectmodel import we_are_translated from pypy.objspace.std.typeobject import MethodCache +from pypy.objspace.std.mapdict import IndexCache def internal_repr(space, w_object): return space.wrap('%r' % (w_object,)) @@ -36,4 +37,17 @@ def reset_method_cache_counter(space): cache = space.fromcache(MethodCache) cache.misses = {} cache.hits = {} + if space.config.objspace.std.withmapdict: + cache = space.fromcache(IndexCache) + cache.misses = {} + cache.hits = {} +def mapdict_cache_counter(space, name): + """Return a tuple (index_cache_hits, index_cache_misses) for lookups + in the mapdict cache with the given attribute name.""" + assert space.config.objspace.std.withmethodcachecounter + assert space.config.objspace.std.withmapdict + cache = space.fromcache(IndexCache) + return space.newtuple([space.newint(cache.hits.get(name, 0)), + space.newint(cache.misses.get(name, 0))]) +mapdict_cache_counter.unwrap_spec = [ObjSpace, str] diff --git a/pypy/module/__pypy__/test/test_special.py b/pypy/module/__pypy__/test/test_special.py index 516247822d..af0cd80210 100644 --- a/pypy/module/__pypy__/test/test_special.py +++ b/pypy/module/__pypy__/test/test_special.py @@ -17,3 +17,7 @@ class AppTest(object): from __pypy__ import isfake import select assert isfake(select) + + def test_cpumodel(self): + import __pypy__ + assert hasattr(__pypy__, 'cpumodel') diff --git a/pypy/module/_rawffi/test/test__rawffi.py b/pypy/module/_rawffi/test/test__rawffi.py index a232b9422e..81d60ec085 100644 --- a/pypy/module/_rawffi/test/test__rawffi.py +++ b/pypy/module/_rawffi/test/test__rawffi.py @@ -296,6 +296,7 @@ class AppTestFfi: assert _rawffi.charp2string(res[0]) is None arg1.free() arg2.free() + a.free() def test_raw_callable(self): import _rawffi diff --git a/pypy/module/_rawffi/test/test_nested.py b/pypy/module/_rawffi/test/test_nested.py index 077dfe3f78..e0bbd87a84 100644 --- a/pypy/module/_rawffi/test/test_nested.py +++ b/pypy/module/_rawffi/test/test_nested.py @@ -107,7 +107,6 @@ class AppTestNested: assert S.fieldoffset('x') == 0 assert S.fieldoffset('ar') == A5alignment s = S() - s = S() s.x = 'G' raises(TypeError, 's.ar') assert s.fieldaddress('ar') == s.buffer + S.fieldoffset('ar') diff --git a/pypy/module/_stackless/test/conftest.py b/pypy/module/_stackless/test/conftest.py new file mode 100644 index 0000000000..1c5e26a206 --- /dev/null +++ b/pypy/module/_stackless/test/conftest.py @@ -0,0 +1,7 @@ +import sys +import py.test + +def pytest_runtest_setup(item): + if sys.platform == 'win32': + py.test.skip("stackless tests segfault on Windows") + diff --git a/pypy/module/array/interp_array.py b/pypy/module/array/interp_array.py index 2c81b7fcff..ff62c17bee 100644 --- a/pypy/module/array/interp_array.py +++ b/pypy/module/array/interp_array.py @@ -27,7 +27,7 @@ def w_array(space, w_cls, typecode, w_args=None): typecode = typecode[0] if space.is_w(w_cls, space.gettypeobject(W_ArrayBase.typedef)): - if len(w_args.keywords_w) > 0: + if w_args.keywords: # XXX this might be forbidden fishing msg = 'array.array() does not take keyword arguments' raise OperationError(space.w_TypeError, space.wrap(msg)) diff --git a/pypy/module/cpyext/api.py b/pypy/module/cpyext/api.py index fc1577c7fa..7805d6f3e2 100644 --- a/pypy/module/cpyext/api.py +++ b/pypy/module/cpyext/api.py @@ -226,7 +226,7 @@ def cpython_api(argtypes, restype, error=_NOT_SPECIFIED, external=True): def unwrapper(space, *args): from pypy.module.cpyext.pyobject import Py_DecRef from pypy.module.cpyext.pyobject import make_ref, from_ref - from pypy.module.cpyext.pyobject import BorrowPair + from pypy.module.cpyext.pyobject import Reference newargs = () to_decref = [] assert len(args) == len(api_function.argtypes) @@ -270,8 +270,8 @@ def cpython_api(argtypes, restype, error=_NOT_SPECIFIED, external=True): return api_function.error_value if res is None: return None - elif isinstance(res, BorrowPair): - return res.w_borrowed + elif isinstance(res, Reference): + return res.get_wrapped(space) else: return res finally: @@ -473,7 +473,7 @@ def make_wrapper(space, callable): @specialize.ll() def wrapper(*args): from pypy.module.cpyext.pyobject import make_ref, from_ref - from pypy.module.cpyext.pyobject import BorrowPair + from pypy.module.cpyext.pyobject import Reference # we hope that malloc removal removes the newtuple() that is # inserted exactly here by the varargs specializer llop.gc_stack_bottom(lltype.Void) # marker for trackgcroot.py @@ -525,7 +525,7 @@ def make_wrapper(space, callable): elif is_PyObject(callable.api_func.restype): if result is None: retval = make_ref(space, None) - elif isinstance(result, BorrowPair): + elif isinstance(result, Reference): retval = result.get_ref(space) elif not rffi._isllptr(result): retval = rffi.cast(callable.api_func.restype, @@ -908,8 +908,10 @@ def load_extension_module(space, path, name): from pypy.rlib import rdynload try: ll_libname = rffi.str2charp(path) - dll = rdynload.dlopen(ll_libname) - lltype.free(ll_libname, flavor='raw') + try: + dll = rdynload.dlopen(ll_libname) + finally: + lltype.free(ll_libname, flavor='raw') except rdynload.DLOpenError, e: raise operationerrfmt( space.w_ImportError, diff --git a/pypy/module/cpyext/cdatetime.py b/pypy/module/cpyext/cdatetime.py index 3081233375..6aa54f695e 100644 --- a/pypy/module/cpyext/cdatetime.py +++ b/pypy/module/cpyext/cdatetime.py @@ -4,7 +4,7 @@ from pypy.module.cpyext.pyobject import PyObject, make_ref, Py_DecRef from pypy.module.cpyext.api import ( cpython_api, CANNOT_FAIL, cpython_struct, PyObjectFields) from pypy.module.cpyext.import_ import PyImport_Import -from pypy.module.cpyext.typeobject import PyTypeObjectPtr +from pypy.module.cpyext.typeobject import PyTypeObjectPtr, render_immortal from pypy.module.cpyext.state import State from pypy.interpreter.error import OperationError from pypy.tool.sourcetools import func_renamer @@ -22,25 +22,34 @@ PyDateTime_CAPI = cpython_struct( @cpython_api([], lltype.Ptr(PyDateTime_CAPI), error=lltype.nullptr(PyDateTime_CAPI)) def _PyDateTime_Import(space): - datetimeAPI = lltype.malloc(PyDateTime_CAPI, flavor='raw') + datetimeAPI = lltype.malloc(PyDateTime_CAPI, flavor='raw', + track_allocation=False) if not we_are_translated(): datetimeAPI_dealloc(space) space.fromcache(State).datetimeAPI = datetimeAPI w_datetime = PyImport_Import(space, space.wrap("datetime")) + w_type = space.getattr(w_datetime, space.wrap("date")) datetimeAPI.c_DateType = rffi.cast( PyTypeObjectPtr, make_ref(space, w_type)) + render_immortal(datetimeAPI.c_DateType, w_type) + w_type = space.getattr(w_datetime, space.wrap("datetime")) datetimeAPI.c_DateTimeType = rffi.cast( PyTypeObjectPtr, make_ref(space, w_type)) + render_immortal(datetimeAPI.c_DateTimeType, w_type) + w_type = space.getattr(w_datetime, space.wrap("time")) datetimeAPI.c_TimeType = rffi.cast( PyTypeObjectPtr, make_ref(space, w_type)) + render_immortal(datetimeAPI.c_TimeType, w_type) + w_type = space.getattr(w_datetime, space.wrap("timedelta")) datetimeAPI.c_DeltaType = rffi.cast( PyTypeObjectPtr, make_ref(space, w_type)) + render_immortal(datetimeAPI.c_DeltaType, w_type) return datetimeAPI diff --git a/pypy/module/cpyext/presetup.py b/pypy/module/cpyext/presetup.py index 5a1605f048..16c205ad79 100644 --- a/pypy/module/cpyext/presetup.py +++ b/pypy/module/cpyext/presetup.py @@ -21,6 +21,8 @@ from distutils import sysconfig from pypy.conftest import gettestobjspace from pypy.module.cpyext.api import build_bridge +from pypy.module.imp.importing import get_so_extension + usemodules = ['cpyext', 'thread'] if sys.platform == 'win32': usemodules.append('_winreg') # necessary in distutils @@ -35,6 +37,7 @@ def get_python_inc(plat_specific=0, prefix=None): def patch_distutils(): sysconfig.get_python_inc = get_python_inc + sysconfig.get_config_vars()['SO'] = get_so_extension(space) patch_distutils() diff --git a/pypy/module/cpyext/pyobject.py b/pypy/module/cpyext/pyobject.py index e596398e5c..a0372dd475 100644 --- a/pypy/module/cpyext/pyobject.py +++ b/pypy/module/cpyext/pyobject.py @@ -424,7 +424,18 @@ def make_borrowed_ref(space, w_container, w_borrowed): state = space.fromcache(RefcountState) return state.make_borrowed(w_container, w_borrowed) -class BorrowPair: +class Reference: + def __init__(self, pyobj): + assert not isinstance(pyobj, W_Root) + self.pyobj = pyobj + + def get_ref(self, space): + return self.pyobj + + def get_wrapped(self, space): + return from_ref(space, self.pyobj) + +class BorrowPair(Reference): """ Delays the creation of a borrowed reference. """ @@ -435,6 +446,9 @@ class BorrowPair: def get_ref(self, space): return make_borrowed_ref(space, self.w_container, self.w_borrowed) + def get_wrapped(self, space): + return self.w_borrowed + def borrow_from(container, borrowed): return BorrowPair(container, borrowed) diff --git a/pypy/module/cpyext/test/test_borrow.py b/pypy/module/cpyext/test/test_borrow.py index 30fc28f4b5..e02a39854d 100644 --- a/pypy/module/cpyext/test/test_borrow.py +++ b/pypy/module/cpyext/test/test_borrow.py @@ -31,6 +31,7 @@ class AppTestBorrow(AppTestCpythonExtensionBase): g = PyTuple_GetItem(t, 0); // borrows reference again printf("Refcnt4: %i\\n", f->ob_refcnt); printf("COMPARE: %i\\n", f == g); + fflush(stdout); Py_DECREF(t); Py_RETURN_TRUE; """), diff --git a/pypy/module/cpyext/test/test_cpyext.py b/pypy/module/cpyext/test/test_cpyext.py index 60e993c6a5..e27435c23b 100644 --- a/pypy/module/cpyext/test/test_cpyext.py +++ b/pypy/module/cpyext/test/test_cpyext.py @@ -42,7 +42,7 @@ class AppTestApi: raises(ImportError, cpyext.load_module, "missing.file", "foo") raises(ImportError, cpyext.load_module, self.libc, "invalid.function") -def compile_module(modname, **kwds): +def compile_module(space, modname, **kwds): """ Build an extension module and return the filename of the resulting native code file. @@ -65,10 +65,8 @@ def compile_module(modname, **kwds): [], eci, outputfilename=str(dirname/modname), standalone=False) - if sys.platform == 'win32': - pydname = soname.new(purebasename=modname, ext='.pyd') - else: - pydname = soname.new(purebasename=modname, ext='.so') + from pypy.module.imp.importing import get_so_extension + pydname = soname.new(purebasename=modname, ext=get_so_extension(space)) soname.rename(pydname) return str(pydname) @@ -153,7 +151,7 @@ class AppTestCpythonExtensionBase(LeakCheckingTest): kwds["link_files"] = [str(api_library + '.so')] if sys.platform == 'linux2': kwds["compile_extra"]=["-Werror=implicit-function-declaration"] - return compile_module(name, **kwds) + return compile_module(self.space, name, **kwds) def import_module(self, name, init=None, body='', load_it=True, filename=None): diff --git a/pypy/module/cpyext/typeobject.py b/pypy/module/cpyext/typeobject.py index 1796ebc374..e400a4ccbf 100644 --- a/pypy/module/cpyext/typeobject.py +++ b/pypy/module/cpyext/typeobject.py @@ -182,10 +182,10 @@ def tp_new_wrapper(space, self, w_args, w_kwds): subtype = rffi.cast(PyTypeObjectPtr, make_ref(space, w_subtype)) try: - obj = generic_cpy_call(space, tp_new, subtype, w_args, w_kwds) + w_obj = generic_cpy_call(space, tp_new, subtype, w_args, w_kwds) finally: Py_DecRef(space, w_subtype) - return obj + return w_obj @specialize.memo() def get_new_method_def(space): @@ -193,10 +193,14 @@ def get_new_method_def(space): if state.new_method_def: return state.new_method_def from pypy.module.cpyext.modsupport import PyMethodDef - ptr = lltype.malloc(PyMethodDef, flavor="raw", zero=True) + ptr = lltype.malloc(PyMethodDef, flavor="raw", zero=True, + immortal=True) ptr.c_ml_name = rffi.str2charp("__new__") + lltype.render_immortal(ptr.c_ml_name) rffi.setintfield(ptr, 'c_ml_flags', METH_VARARGS | METH_KEYWORDS) - ptr.c_ml_doc = rffi.str2charp("T.__new__(S, ...) -> a new object with type S, a subtype of T") + ptr.c_ml_doc = rffi.str2charp( + "T.__new__(S, ...) -> a new object with type S, a subtype of T") + lltype.render_immortal(ptr.c_ml_doc) state.new_method_def = ptr return ptr @@ -429,6 +433,12 @@ def type_attach(space, py_obj, w_type): finish_type_1(space, pto) finish_type_2(space, pto, w_type) + if space.type(w_type).is_cpytype(): + # XXX Types with a C metatype are never freed, try to see why... + render_immortal(pto, w_type) + lltype.render_immortal(pto) + lltype.render_immortal(pto.c_tp_name) + pto.c_tp_basicsize = rffi.sizeof(typedescr.basestruct) if pto.c_tp_base: if pto.c_tp_base.c_tp_basicsize > pto.c_tp_basicsize: @@ -534,12 +544,25 @@ def type_realize(space, py_obj): w_obj.ready() finish_type_2(space, py_type, w_obj) + render_immortal(py_type, w_obj) state = space.fromcache(RefcountState) state.non_heaptypes_w.append(w_obj) return w_obj +def render_immortal(py_type, w_obj): + lltype.render_immortal(py_type.c_tp_bases) + lltype.render_immortal(py_type.c_tp_mro) + + assert isinstance(w_obj, W_TypeObject) + if w_obj.is_cpytype(): + lltype.render_immortal(py_type.c_tp_dict) + else: + lltype.render_immortal(py_type.c_tp_name) + if not w_obj.is_cpytype() and w_obj.is_heaptype(): + lltype.render_immortal(py_type) + def finish_type_1(space, pto): """ Sets up tp_bases, necessary before creating the interpreter type. diff --git a/pypy/module/gc/__init__.py b/pypy/module/gc/__init__.py index e76bbfaad3..e003d7b683 100644 --- a/pypy/module/gc/__init__.py +++ b/pypy/module/gc/__init__.py @@ -29,6 +29,7 @@ class Module(MixedModule): 'get_referents': 'referents.get_referents', 'get_referrers': 'referents.get_referrers', '_dump_rpy_heap': 'referents._dump_rpy_heap', + 'get_typeids_z': 'referents.get_typeids_z', 'GcRef': 'referents.W_GcRef', }) MixedModule.__init__(self, space, w_name) diff --git a/pypy/module/gc/app_referents.py b/pypy/module/gc/app_referents.py index 461cb989ac..b97a5baa05 100644 --- a/pypy/module/gc/app_referents.py +++ b/pypy/module/gc/app_referents.py @@ -14,11 +14,25 @@ def dump_rpy_heap(file): and [addr1]..[addrn] are addresses of other objects that this object points to. The full dump is a list of such objects, with a marker [0][0][0][-1] inserted after all GC roots, before all non-roots. + + If the argument is a filename and the 'zlib' module is available, + we also write a 'typeids.txt' in the same directory, if none exists. """ if isinstance(file, str): f = open(file, 'wb') gc._dump_rpy_heap(f.fileno()) f.close() + try: + import zlib, os + except ImportError: + pass + else: + filename2 = os.path.join(os.path.dirname(file), 'typeids.txt') + if not os.path.exists(filename2): + data = zlib.decompress(gc.get_typeids_z()) + f = open(filename2, 'wb') + f.write(data) + f.close() else: if isinstance(file, int): fd = file diff --git a/pypy/module/gc/interp_gc.py b/pypy/module/gc/interp_gc.py index 541316c494..a22dd1e644 100644 --- a/pypy/module/gc/interp_gc.py +++ b/pypy/module/gc/interp_gc.py @@ -10,6 +10,10 @@ def collect(space): from pypy.objspace.std.typeobject import MethodCache cache = space.fromcache(MethodCache) cache.clear() + if space.config.objspace.std.withmapdict: + from pypy.objspace.std.mapdict import IndexCache + cache = space.fromcache(IndexCache) + cache.clear() rgc.collect() return space.wrap(0) diff --git a/pypy/module/gc/referents.py b/pypy/module/gc/referents.py index 0aba55771b..8233419611 100644 --- a/pypy/module/gc/referents.py +++ b/pypy/module/gc/referents.py @@ -177,3 +177,9 @@ def _dump_rpy_heap(space, fd): if not ok: raise missing_operation(space) _dump_rpy_heap.unwrap_spec = [ObjSpace, int] + +def get_typeids_z(space): + a = rgc.get_typeids_z() + s = ''.join([a[i] for i in range(len(a))]) + return space.wrap(s) +get_typeids_z.unwrap_spec = [ObjSpace] diff --git a/pypy/module/gc/test/test_gc.py b/pypy/module/gc/test/test_gc.py index dcca7a853f..295fcf1813 100644 --- a/pypy/module/gc/test/test_gc.py +++ b/pypy/module/gc/test/test_gc.py @@ -117,6 +117,33 @@ class AppTestGcMethodCache(object): pass C().f() # Fill the method cache rlist.append(weakref.ref(C)) + for i in range(10): + f() + gc.collect() # the classes C should all go away here + # the last class won't go in mapdict, as long as the code object of f + # is around + rlist.pop() + for r in rlist: + assert r() is None + +class AppTestGcMapDictIndexCache(AppTestGcMethodCache): + def setup_class(cls): + cls.space = gettestobjspace(**{"objspace.std.withmethodcache": True, + "objspace.std.withmapdict": True}) + + + def test_clear_index_cache(self): + import gc, weakref + rlist = [] + def f(): + class C(object): + def f(self): + pass + c = C() + c.x = 1 + getattr(c, "x") # fill the index cache without using the local cache + getattr(c, "x") + rlist.append(weakref.ref(C)) for i in range(5): f() gc.collect() # the classes C should all go away here diff --git a/pypy/module/imp/importing.py b/pypy/module/imp/importing.py index b69c621e42..bb0314fb50 100644 --- a/pypy/module/imp/importing.py +++ b/pypy/module/imp/importing.py @@ -12,7 +12,7 @@ from pypy.interpreter.eval import Code from pypy.rlib import streamio, jit from pypy.rlib.streamio import StreamErrors from pypy.rlib.rarithmetic import intmask -from pypy.rlib.objectmodel import we_are_translated +from pypy.rlib.objectmodel import we_are_translated, specialize SEARCH_ERROR = 0 PY_SOURCE = 1 @@ -25,10 +25,26 @@ PY_FROZEN = 7 # PY_CODERESOURCE = 8 IMP_HOOK = 9 -if sys.platform.startswith('win'): - so_extension = ".pyd" +if sys.platform == 'win32': + SO = ".pyd" else: - so_extension = ".so" + SO = ".so" +DEFAULT_SOABI = 'pypy-14' + +@specialize.memo() +def get_so_extension(space): + if space.config.objspace.soabi is not None: + soabi = space.config.objspace.soabi + else: + soabi = DEFAULT_SOABI + + if not soabi: + return SO + + if not space.config.translating: + soabi += 'i' + + return '.' + soabi + SO def find_modtype(space, filepart): """Check which kind of module to import for the given filepart, @@ -53,6 +69,7 @@ def find_modtype(space, filepart): return PY_COMPILED, ".pyc", "rb" if space.config.objspace.usemodules.cpyext: + so_extension = get_so_extension(space) pydfile = filepart + so_extension if os.path.exists(pydfile) and case_ok(pydfile): return C_EXTENSION, so_extension, "rb" diff --git a/pypy/module/imp/interp_imp.py b/pypy/module/imp/interp_imp.py index e94453b1dc..fea66141b4 100644 --- a/pypy/module/imp/interp_imp.py +++ b/pypy/module/imp/interp_imp.py @@ -8,10 +8,16 @@ import struct def get_suffixes(space): w = space.wrap - return space.newlist([ + suffixes_w = [] + if space.config.objspace.usemodules.cpyext: + suffixes_w.append( + space.newtuple([w(importing.get_so_extension(space)), + w('rb'), w(importing.C_EXTENSION)])) + suffixes_w.extend([ space.newtuple([w('.py'), w('U'), w(importing.PY_SOURCE)]), space.newtuple([w('.pyc'), w('rb'), w(importing.PY_COMPILED)]), ]) + return space.newlist(suffixes_w) def get_magic(space): x = importing.get_pyc_magic(space) diff --git a/pypy/module/imp/test/test_app.py b/pypy/module/imp/test/test_app.py index e96588fbf2..49b3ad5f0f 100644 --- a/pypy/module/imp/test/test_app.py +++ b/pypy/module/imp/test/test_app.py @@ -47,6 +47,9 @@ class AppTestImpModule: elif mode == self.imp.PY_COMPILED: assert suffix in ('.pyc', '.pyo') assert type == 'rb' + elif mode == self.imp.C_EXTENSION: + assert suffix.endswith(('.pyd', '.so')) + assert type == 'rb' def test_obscure_functions(self): diff --git a/pypy/module/imp/test/test_import.py b/pypy/module/imp/test/test_import.py index 056fe9caf9..5a27ab84df 100644 --- a/pypy/module/imp/test/test_import.py +++ b/pypy/module/imp/test/test_import.py @@ -473,6 +473,17 @@ class AppTestImport: except ImportError: pass +class TestAbi: + def test_abi_tag(self): + space1 = gettestobjspace(soabi='TEST') + space2 = gettestobjspace(soabi='') + if sys.platform == 'win32': + assert importing.get_so_extension(space1) == '.TESTi.pyd' + assert importing.get_so_extension(space2) == '.pyd' + else: + assert importing.get_so_extension(space1) == '.TESTi.so' + assert importing.get_so_extension(space2) == '.so' + def _getlong(data): x = marshal.dumps(data) return x[-4:] diff --git a/pypy/module/posix/interp_posix.py b/pypy/module/posix/interp_posix.py index 47c98ad319..042e20f16f 100644 --- a/pypy/module/posix/interp_posix.py +++ b/pypy/module/posix/interp_posix.py @@ -454,7 +454,8 @@ class State: self.w_environ = space.newdict() if _WIN: self.cryptProviderPtr = lltype.malloc( - rffi.CArray(HCRYPTPROV), 1, zero=True, flavor='raw') + rffi.CArray(HCRYPTPROV), 1, zero=True, + flavor='raw', immortal=True) def startup(self, space): _convertenviron(space, self.w_environ) def _freeze_(self): diff --git a/pypy/module/pypyjit/interp_jit.py b/pypy/module/pypyjit/interp_jit.py index e2a71a04b4..77057a1879 100644 --- a/pypy/module/pypyjit/interp_jit.py +++ b/pypy/module/pypyjit/interp_jit.py @@ -6,6 +6,7 @@ This is transformed to become a JIT by code elsewhere: pypy/jit/* from pypy.tool.pairtype import extendabletype from pypy.rlib.rarithmetic import r_uint, intmask from pypy.rlib.jit import JitDriver, hint, we_are_jitted, dont_look_inside +from pypy.rlib.jit import current_trace_length import pypy.interpreter.pyopcode # for side-effects from pypy.interpreter.error import OperationError, operationerrfmt from pypy.interpreter.gateway import ObjSpace, Arguments, W_Root @@ -80,9 +81,22 @@ class __extend__(PyFrame): def jump_absolute(self, jumpto, _, ec=None): if we_are_jitted(): + # Normally, the tick counter is decremented by 100 for every + # Python opcode. Here, to better support JIT compilation of + # small loops, we decrement it by a possibly smaller constant. + # We get the maximum 100 when the (unoptimized) trace length + # is at least 3200 (a bit randomly). + trace_length = r_uint(current_trace_length()) + decr_by = trace_length // 32 + if decr_by < 1: + decr_by = 1 + elif decr_by > 100: # also if current_trace_length() returned -1 + decr_by = 100 + # self.last_instr = intmask(jumpto) - ec.bytecode_trace(self) + ec.bytecode_trace(self, intmask(decr_by)) jumpto = r_uint(self.last_instr) + # pypyjitdriver.can_enter_jit(frame=self, ec=ec, next_instr=jumpto, pycode=self.getcode()) return jumpto diff --git a/pypy/module/pypyjit/policy.py b/pypy/module/pypyjit/policy.py index 20a6452b2a..d80893476c 100644 --- a/pypy/module/pypyjit/policy.py +++ b/pypy/module/pypyjit/policy.py @@ -6,7 +6,8 @@ class PyPyJitPolicy(JitPolicy): if (modname == '__builtin__.operation' or modname == '__builtin__.abstractinst' or modname == '__builtin__.interp_classobj' or - modname == '__builtin__.functional'): + modname == '__builtin__.functional' or + modname == '__builtin__.descriptor'): return True if '.' in modname: modname, _ = modname.split('.', 1) @@ -19,7 +20,7 @@ class PyPyJitPolicy(JitPolicy): # this function should never actually return True directly # but instead call the base implementation mod = func.__module__ or '?' - + if mod.startswith('pypy.objspace.'): # gc_id operation if func.__name__ == 'id__ANY': @@ -36,5 +37,5 @@ class PyPyJitPolicy(JitPolicy): modname = mod[len('pypy.module.'):] if not self.look_inside_pypy_module(modname): return False - + return True diff --git a/pypy/module/pypyjit/test/test_policy.py b/pypy/module/pypyjit/test/test_policy.py index 02ac51ae06..1f781e10de 100644 --- a/pypy/module/pypyjit/test/test_policy.py +++ b/pypy/module/pypyjit/test/test_policy.py @@ -12,7 +12,7 @@ def test_bigint(): def test_rlocale(): from pypy.rlib.rlocale import setlocale - assert not pypypolicy.look_inside_function(setlocale) + assert not pypypolicy.look_inside_function(setlocale) def test_geninterp(): d = {'_geninterp_': True} @@ -28,6 +28,10 @@ def test_pyparser(): from pypy.interpreter.pyparser import parser assert not pypypolicy.look_inside_function(parser.Grammar.__init__.im_func) +def test_property(): + from pypy.module.__builtin__.descriptor import W_Property + assert pypypolicy.look_inside_function(W_Property.get.im_func) + def test_pypy_module(): from pypy.module._random.interp_random import W_Random assert not pypypolicy.look_inside_function(W_Random.random) @@ -35,6 +39,7 @@ def test_pypy_module(): assert pypypolicy.look_inside_pypy_module('__builtin__.operation') assert pypypolicy.look_inside_pypy_module('__builtin__.abstractinst') assert pypypolicy.look_inside_pypy_module('__builtin__.functional') + assert pypypolicy.look_inside_pypy_module('__builtin__.descriptor') assert pypypolicy.look_inside_pypy_module('exceptions.interp_exceptions') for modname in 'pypyjit', 'signal', 'micronumpy', 'math', 'imp': assert pypypolicy.look_inside_pypy_module(modname) @@ -42,4 +47,3 @@ def test_pypy_module(): def test_see_jit_module(): assert pypypolicy.look_inside_pypy_module('pypyjit.interp_jit') - diff --git a/pypy/module/pypyjit/test/test_pypy_c.py b/pypy/module/pypyjit/test/test_pypy_c.py index 43550aa6b6..c9be155293 100644 --- a/pypy/module/pypyjit/test/test_pypy_c.py +++ b/pypy/module/pypyjit/test/test_pypy_c.py @@ -422,10 +422,75 @@ class PyPyCJITTests(object): ([1000], 49500), ([10000], 495000), ([100000], 4950000)) - assert len(self.loops) == 2 + assert len(self.loops) == 3 op, = self.get_by_bytecode("CALL_FUNCTION_KW") # XXX a bit too many guards, but better than before - assert len(op.get_opnames("guard")) <= 10 + assert len(op.get_opnames("guard")) <= 12 + + def test_stararg_virtual(self): + self.run_source(''' + d = {} + + def g(*args): + return len(args) + def h(a, b, c): + return c + + def main(x): + s = 0 + for i in range(x): + l = [i, x, 2] + s += g(*l) + s += h(*l) + s += g(i, x, 2) + for i in range(x): + l = [x, 2] + s += g(i, *l) + s += h(i, *l) + return s + ''', 100000, ([100], 1300), + ([1000], 13000), + ([10000], 130000), + ([100000], 1300000)) + assert len(self.loops) == 2 + ops = self.get_by_bytecode("CALL_FUNCTION_VAR") + assert len(ops) == 4 + for op in ops: + assert len(op.get_opnames("new")) == 0 + assert len(op.get_opnames("call_may_force")) == 0 + + ops = self.get_by_bytecode("CALL_FUNCTION") + for op in ops: + assert len(op.get_opnames("new")) == 0 + assert len(op.get_opnames("call_may_force")) == 0 + + def test_stararg(self): + self.run_source(''' + d = {} + + def g(*args): + return args[-1] + def h(*args): + return len(args) + + def main(x): + s = 0 + l = [] + i = 0 + while i < x: + l.append(1) + s += g(*l) + i = h(*l) + return s + ''', 100000, ([100], 100), + ([1000], 1000), + ([2000], 2000), + ([4000], 4000)) + assert len(self.loops) == 1 + ops = self.get_by_bytecode("CALL_FUNCTION_VAR") + for op in ops: + assert len(op.get_opnames("new_with_vtable")) == 0 + assert len(op.get_opnames("call_may_force")) == 0 def test_virtual_instance(self): self.run_source(''' diff --git a/pypy/module/rctime/interp_time.py b/pypy/module/rctime/interp_time.py index 9a06bef3f9..dbb51df6d8 100644 --- a/pypy/module/rctime/interp_time.py +++ b/pypy/module/rctime/interp_time.py @@ -69,7 +69,7 @@ if _POSIX: CLOCKS_PER_SEC = cConfig.CLOCKS_PER_SEC clock_t = cConfig.clock_t tm = cConfig.tm -glob_buf = lltype.malloc(tm, flavor='raw', zero=True) +glob_buf = lltype.malloc(tm, flavor='raw', zero=True, immortal=True) if cConfig.has_gettimeofday: c_gettimeofday = external('gettimeofday', [rffi.VOIDP, rffi.VOIDP], rffi.INT) diff --git a/pypy/module/signal/__init__.py b/pypy/module/signal/__init__.py index 613de46930..a861feddff 100644 --- a/pypy/module/signal/__init__.py +++ b/pypy/module/signal/__init__.py @@ -34,9 +34,7 @@ class Module(MixedModule): MixedModule.__init__(self, space, *args) # add the signal-checking callback as an action on the space space.check_signal_action = interp_signal.CheckSignalAction(space) - space.actionflag.register_action(space.check_signal_action) - # use the C-level pypysig_occurred variable as the action flag - # (the result is that the C-level signal handler will directly - # set the flag for the CheckSignalAction) + space.actionflag.register_periodic_action(space.check_signal_action, + use_bytecode_counter=False) space.actionflag.__class__ = interp_signal.SignalActionFlag # xxx yes I know the previous line is a hack diff --git a/pypy/module/signal/interp_signal.py b/pypy/module/signal/interp_signal.py index 26efaecfb2..73b1904075 100644 --- a/pypy/module/signal/interp_signal.py +++ b/pypy/module/signal/interp_signal.py @@ -1,6 +1,7 @@ from pypy.interpreter.error import OperationError from pypy.interpreter.baseobjspace import W_Root, ObjSpace from pypy.interpreter.executioncontext import AsyncAction, AbstractActionFlag +from pypy.interpreter.executioncontext import PeriodicAsyncAction import signal as cpy_signal from pypy.rpython.lltypesystem import lltype, rffi from pypy.translator.tool.cbuild import ExternalCompilationInfo @@ -52,19 +53,29 @@ c_pause = external('pause', [], rffi.INT) class SignalActionFlag(AbstractActionFlag): - def get(self): + # This class uses the C-level pypysig_counter variable as the tick + # counter. The C-level signal handler will reset it to -1 whenever + # a signal is received. + + def get_ticker(self): p = pypysig_getaddr_occurred() return p.c_value - def set(self, value): + + def reset_ticker(self, value): p = pypysig_getaddr_occurred() p.c_value = value + def decrement_ticker(self, by): + p = pypysig_getaddr_occurred() + value = p.c_value + if self.has_bytecode_counter: # this 'if' is constant-folded + value -= by + p.c_value = value + return value -class CheckSignalAction(AsyncAction): - """An action that is automatically invoked when a signal is received.""" - # The C-level signal handler sets the bit 30 of pypysig_occurred: - bitmask = 1 << 30 +class CheckSignalAction(PeriodicAsyncAction): + """An action that is automatically invoked when a signal is received.""" def __init__(self, space): AsyncAction.__init__(self, space) @@ -73,7 +84,6 @@ class CheckSignalAction(AsyncAction): # need a helper action in case signals arrive in a non-main thread self.pending_signals = {} self.reissue_signal_action = ReissueSignalAction(space) - space.actionflag.register_action(self.reissue_signal_action) else: self.reissue_signal_action = None diff --git a/pypy/module/signal/test/test_signal.py b/pypy/module/signal/test/test_signal.py index 10c29a531f..729c404a8b 100644 --- a/pypy/module/signal/test/test_signal.py +++ b/pypy/module/signal/test/test_signal.py @@ -1,6 +1,38 @@ import os, py +import signal as cpy_signal from pypy.conftest import gettestobjspace + +class TestCheckSignals: + + def setup_class(cls): + if not hasattr(os, 'kill') or not hasattr(os, 'getpid'): + py.test.skip("requires os.kill() and os.getpid()") + cls.space = gettestobjspace(usemodules=['signal']) + + def test_checksignals(self): + space = self.space + w_received = space.appexec([], """(): + import signal + received = [] + def myhandler(signum, frame): + received.append(signum) + signal.signal(signal.SIGUSR1, myhandler) + return received""") + # + assert not space.is_true(w_received) + # + # send the signal now + os.kill(os.getpid(), cpy_signal.SIGUSR1) + # + # myhandler() should not be immediately called + assert not space.is_true(w_received) + # + # calling ec.checksignals() should call it + space.getexecutioncontext().checksignals() + assert space.is_true(w_received) + + class AppTestSignal: def setup_class(cls): @@ -24,18 +56,12 @@ class AppTestSignal: signal.signal(signal.SIGUSR1, myhandler) posix.kill(posix.getpid(), signal.SIGUSR1) - for i in range(10000): - # wait a bit for the signal to be delivered to the handler - if received: - break + # the signal should be delivered to the handler immediately assert received == [signal.SIGUSR1] del received[:] posix.kill(posix.getpid(), signal.SIGUSR1) - for i in range(10000): - # wait a bit for the signal to be delivered to the handler - if received: - break + # the signal should be delivered to the handler immediately assert received == [signal.SIGUSR1] del received[:] diff --git a/pypy/module/sys/__init__.py b/pypy/module/sys/__init__.py index 7e3b8ab9ba..492ad1560e 100644 --- a/pypy/module/sys/__init__.py +++ b/pypy/module/sys/__init__.py @@ -10,7 +10,6 @@ class Module(MixedModule): if space.config.translating: del self.__class__.interpleveldefs['pypy_getudir'] super(Module, self).__init__(space, w_name) - self.checkinterval = 100 self.recursionlimit = 100 self.w_default_encoder = None self.defaultencoding = "ascii" diff --git a/pypy/module/sys/version.py b/pypy/module/sys/version.py index 4b6bd08fd4..6d80545fee 100644 --- a/pypy/module/sys/version.py +++ b/pypy/module/sys/version.py @@ -8,7 +8,7 @@ import os CPYTHON_VERSION = (2, 5, 2, "beta", 42) #XXX # sync patchlevel.h CPYTHON_API_VERSION = 1012 #XXX # sync with include/modsupport.h -PYPY_VERSION = (1, 3, 0, "beta", '?') #XXX # sync patchlevel.h +PYPY_VERSION = (1, 4, 0, "beta", '?') #XXX # sync patchlevel.h # the last item is replaced by the svn revision ^^^ TRIM_URL_UP_TO = 'svn/pypy/' diff --git a/pypy/module/sys/vm.py b/pypy/module/sys/vm.py index 154cce149e..a21324aac0 100644 --- a/pypy/module/sys/vm.py +++ b/pypy/module/sys/vm.py @@ -67,7 +67,7 @@ def getrecursionlimit(space): def setcheckinterval(space, interval): """Tell the Python interpreter to check for asynchronous events every n instructions. This also affects how often thread switches occur.""" - space.actionflag.setcheckinterval(space, interval) + space.actionflag.setcheckinterval(interval) setcheckinterval.unwrap_spec = [ObjSpace, int] def getcheckinterval(space): @@ -77,7 +77,7 @@ def getcheckinterval(space): # return 0. The idea is that according to the CPython docs, <= 0 # means "check every virtual instruction, maximizing responsiveness # as well as overhead". - result = space.sys.checkinterval + result = space.actionflag.getcheckinterval() if result <= 1: result = 0 return space.wrap(result) diff --git a/pypy/module/thread/gil.py b/pypy/module/thread/gil.py index 87fee7de2c..500ca17058 100644 --- a/pypy/module/thread/gil.py +++ b/pypy/module/thread/gil.py @@ -20,7 +20,8 @@ class GILThreadLocals(OSThreadLocals): def initialize(self, space): # add the GIL-releasing callback as an action on the space - space.actionflag.register_action(GILReleaseAction(space)) + space.actionflag.register_periodic_action(GILReleaseAction(space), + use_bytecode_counter=True) def setup_threads(self, space): """Enable threads in the object space, if they haven't already been.""" @@ -44,7 +45,6 @@ class GILThreadLocals(OSThreadLocals): # test_compile_lock. As a workaround, we repatch these global # fields systematically. spacestate.ll_GIL = self.ll_GIL - spacestate.actionflag = space.actionflag invoke_around_extcall(before_external_call, after_external_call) return result @@ -68,18 +68,17 @@ class SpaceState: def _freeze_(self): self.ll_GIL = thread.null_ll_lock - self.actionflag = None - self.set_actionflag_bit_after_thread_switch = 0 + self.action_after_thread_switch = None + # ^^^ set by AsyncAction.fire_after_thread_switch() return False def after_thread_switch(self): # this is support logic for the signal module, to help it deliver # signals to the main thread. - actionflag = self.actionflag - if actionflag is not None: - flag = actionflag.get() - flag |= self.set_actionflag_bit_after_thread_switch - actionflag.set(flag) + action = self.action_after_thread_switch + if action is not None: + self.action_after_thread_switch = None + action.fire() spacestate = SpaceState() spacestate._freeze_() diff --git a/pypy/module/thread/test/test_gil.py b/pypy/module/thread/test/test_gil.py index b2dacf0a06..03db6e7f92 100644 --- a/pypy/module/thread/test/test_gil.py +++ b/pypy/module/thread/test/test_gil.py @@ -8,7 +8,7 @@ class FakeEC(object): pass class FakeActionFlag(object): - def register_action(self, action): + def register_periodic_action(self, action, use_bytecode_counter): pass def get(self): return 0 diff --git a/pypy/objspace/flow/model.py b/pypy/objspace/flow/model.py index f7c33ad534..ea574e141d 100644 --- a/pypy/objspace/flow/model.py +++ b/pypy/objspace/flow/model.py @@ -355,7 +355,7 @@ class SpaceOperation(object): return "%r = %s(%s)" % (self.result, self.opname, ", ".join(map(repr, self.args))) -class Atom: +class Atom(object): def __init__(self, name): self.__name__ = name # make save_global happy def __repr__(self): diff --git a/pypy/objspace/flow/objspace.py b/pypy/objspace/flow/objspace.py index 12190a3791..11bb0c1788 100644 --- a/pypy/objspace/flow/objspace.py +++ b/pypy/objspace/flow/objspace.py @@ -210,6 +210,11 @@ class FlowObjSpace(ObjSpace): check_class = self.unwrap(w_check_class) except UnwrapException: raise Exception, "non-constant except guard" + if check_class in (NotImplementedError, AssertionError): + # if we are in geninterp, we cannot catch these exceptions + if not self.config.translation.builtins_can_raise_exceptions: + raise error.FlowingError("Catching %s is not valid in RPython" % + check_class.__name__) if not isinstance(check_class, tuple): # the simple case return ObjSpace.exception_match(self, w_exc_type, w_check_class) diff --git a/pypy/objspace/flow/test/test_objspace.py b/pypy/objspace/flow/test/test_objspace.py index 0a6659e6fe..be5b8824c8 100644 --- a/pypy/objspace/flow/test/test_objspace.py +++ b/pypy/objspace/flow/test/test_objspace.py @@ -5,7 +5,7 @@ from pypy.objspace.flow.model import Constant, Block, Link, Variable, traverse from pypy.objspace.flow.model import flatten, mkentrymap, c_last_exception from pypy.interpreter.argument import Arguments from pypy.translator.simplify import simplify_graph -from pypy.objspace.flow.objspace import FlowObjSpace +from pypy.objspace.flow.objspace import FlowObjSpace, error from pypy.objspace.flow import objspace, flowcontext from pypy import conftest from pypy.tool.stdlib_opcode import bytecode_spec @@ -953,6 +953,22 @@ class TestFlowObjSpace(Base): assert op.args[0] == Constant(g) + def test_cannot_catch_special_exceptions(self): + def f(): + try: + f() + except NotImplementedError: + pass + py.test.raises(error.FlowingError, "self.codetest(f)") + # + def f(): + try: + f() + except AssertionError: + pass + py.test.raises(error.FlowingError, "self.codetest(f)") + + class TestFlowObjSpaceDelay(Base): def setup_class(cls): cls.space = FlowObjSpace() @@ -1013,6 +1029,15 @@ class TestGenInterpStyle(Base): expected.sort() assert excfound == expected + def test_can_catch_special_exceptions(self): + def f(): + try: + f() + except NotImplementedError: + pass + graph = self.codetest(f) + # assert did not crash + DATA = {'x': 5, 'y': 6} diff --git a/pypy/objspace/std/callmethod.py b/pypy/objspace/std/callmethod.py index 54d58e7c37..cffe33e48e 100644 --- a/pypy/objspace/std/callmethod.py +++ b/pypy/objspace/std/callmethod.py @@ -13,6 +13,9 @@ like: (on the left, without the new bytecodes; on the right, with them) from pypy.interpreter import function from pypy.objspace.descroperation import object_getattribute from pypy.rlib import jit, rstack # for resume points +from pypy.objspace.std.mapdict import LOOKUP_METHOD_mapdict, \ + LOOKUP_METHOD_mapdict_fill_cache_method + # This module exports two extra methods for StdObjSpaceFrame implementing # the LOOKUP_METHOD and CALL_METHOD opcodes in an efficient way, as well @@ -30,6 +33,13 @@ def LOOKUP_METHOD(f, nameindex, *ignored): # space = f.space w_obj = f.popvalue() + + if space.config.objspace.std.withmapdict and not jit.we_are_jitted(): + # mapdict has an extra-fast version of this function + from pypy.objspace.std.mapdict import LOOKUP_METHOD_mapdict + if LOOKUP_METHOD_mapdict(f, nameindex, w_obj): + return + w_name = f.getname_w(nameindex) w_value = None @@ -50,6 +60,11 @@ def LOOKUP_METHOD(f, nameindex, *ignored): # nothing in the instance f.pushvalue(w_descr) f.pushvalue(w_obj) + if (space.config.objspace.std.withmapdict and + not jit.we_are_jitted()): + # let mapdict cache stuff + LOOKUP_METHOD_mapdict_fill_cache_method( + f.getcode(), nameindex, w_obj, w_type, w_descr) return if w_value is None: w_value = space.getattr(w_obj, w_name) diff --git a/pypy/objspace/std/mapdict.py b/pypy/objspace/std/mapdict.py index b730234a40..e6a752c7e1 100644 --- a/pypy/objspace/std/mapdict.py +++ b/pypy/objspace/std/mapdict.py @@ -1,4 +1,5 @@ from pypy.rlib import jit, objectmodel, debug +from pypy.rlib.rarithmetic import intmask, r_uint from pypy.interpreter.baseobjspace import W_Root from pypy.objspace.std.dictmultiobject import W_DictMultiObject @@ -15,24 +16,70 @@ NUM_DIGITS_POW2 = 1 << NUM_DIGITS # we want to propagate knowledge that the result cannot be negative class AbstractAttribute(object): - _immutable_fields_ = ['w_cls'] + _immutable_fields_ = ['terminator'] cache_attrs = None _size_estimate = 0 - def __init__(self, space, w_cls): + def __init__(self, space, terminator): self.space = space - self.w_cls = w_cls + assert isinstance(terminator, Terminator) + self.terminator = terminator def read(self, obj, selector): - raise NotImplementedError("abstract base class") + index = self.index(selector) + if index < 0: + return self.terminator._read_terminator(obj, selector) + return obj._mapdict_read_storage(index) def write(self, obj, selector, w_value): - raise NotImplementedError("abstract base class") + index = self.index(selector) + if index < 0: + return self.terminator._write_terminator(obj, selector, w_value) + obj._mapdict_write_storage(index, w_value) + return True def delete(self, obj, selector): return None def index(self, selector): + if (self.space.config.objspace.std.withmethodcache and + not jit.we_are_jitted()): + return self._index_cache(selector) + else: + return self._index(selector) + + @jit.dont_look_inside + def _index_cache(self, selector): + space = self.space + cache = space.fromcache(IndexCache) + SHIFT2 = r_uint.BITS - space.config.objspace.std.methodcachesizeexp + SHIFT1 = SHIFT2 - 5 + attrs_as_int = objectmodel.current_object_addr_as_int(self) + # ^^^Note: see comment in typeobject.py for + # _pure_lookup_where_with_method_cache() + hash_selector = objectmodel.compute_hash(selector) + product = intmask(attrs_as_int * hash_selector) + index_hash = (r_uint(product) ^ (r_uint(product) << SHIFT1)) >> SHIFT2 + # ^^^Note2: same comment too + cached_attr = cache.attrs[index_hash] + if cached_attr is self: + cached_selector = cache.selectors[index_hash] + if cached_selector == selector: + index = cache.indices[index_hash] + if space.config.objspace.std.withmethodcachecounter: + name = selector[0] + cache.hits[name] = cache.hits.get(name, 0) + 1 + return index + index = self._index(selector) + cache.attrs[index_hash] = self + cache.selectors[index_hash] = selector + cache.indices[index_hash] = index + if space.config.objspace.std.withmethodcachecounter: + name = selector[0] + cache.misses[name] = cache.misses.get(name, 0) + 1 + return index + + def _index(self, selector): return -1 def copy(self, obj): @@ -42,7 +89,7 @@ class AbstractAttribute(object): raise NotImplementedError("abstract base class") def get_terminator(self): - raise NotImplementedError("abstract base class") + return self.terminator def set_terminator(self, obj, terminator): raise NotImplementedError("abstract base class") @@ -95,15 +142,20 @@ class AbstractAttribute(object): raise NotImplementedError("abstract base class") def __repr__(self): - return "<%s w_cls=%s>" % (self.__class__.__name__, self.w_cls) + return "<%s>" % (self.__class__.__name__,) class Terminator(AbstractAttribute): + _immutable_fields_ = ['w_cls'] - def read(self, obj, selector): + def __init__(self, space, w_cls): + AbstractAttribute.__init__(self, space, self) + self.w_cls = w_cls + + def _read_terminator(self, obj, selector): return None - def write(self, obj, selector, w_value): + def _write_terminator(self, obj, selector, w_value): obj._get_mapdict_map().add_attr(obj, selector, w_value) return True @@ -116,9 +168,6 @@ class Terminator(AbstractAttribute): def length(self): return 0 - def get_terminator(self): - return self - def set_terminator(self, obj, terminator): result = Object() result.space = self.space @@ -128,6 +177,9 @@ class Terminator(AbstractAttribute): def remove_dict_entries(self, obj): return self.copy(obj) + def __repr__(self): + return "<%s w_cls=%s>" % (self.__class__.__name__, self.w_cls) + class DictTerminator(Terminator): _immutable_fields_ = ['devolved_dict_terminator'] def __init__(self, space, w_cls): @@ -142,27 +194,27 @@ class DictTerminator(Terminator): class NoDictTerminator(Terminator): - def write(self, obj, selector, w_value): + def _write_terminator(self, obj, selector, w_value): if selector[1] == DICT: return False - return Terminator.write(self, obj, selector, w_value) + return Terminator._write_terminator(self, obj, selector, w_value) class DevolvedDictTerminator(Terminator): - def read(self, obj, selector): + def _read_terminator(self, obj, selector): if selector[1] == DICT: w_dict = obj.getdict() space = self.space return space.finditem_str(w_dict, selector[0]) - return Terminator.read(self, obj, selector) + return Terminator._read_terminator(self, obj, selector) - def write(self, obj, selector, w_value): + def _write_terminator(self, obj, selector, w_value): if selector[1] == DICT: w_dict = obj.getdict() space = self.space space.setitem_str(w_dict, selector[0], w_value) return True - return Terminator.write(self, obj, selector, w_value) + return Terminator._write_terminator(self, obj, selector, w_value) def delete(self, obj, selector): from pypy.interpreter.error import OperationError @@ -189,7 +241,7 @@ class DevolvedDictTerminator(Terminator): class PlainAttribute(AbstractAttribute): _immutable_fields_ = ['selector', 'position', 'back'] def __init__(self, selector, back): - AbstractAttribute.__init__(self, back.space, back.w_cls) + AbstractAttribute.__init__(self, back.space, back.terminator) self.selector = selector self.position = back.length() self.back = back @@ -199,17 +251,6 @@ class PlainAttribute(AbstractAttribute): w_value = self.read(obj, self.selector) new_obj._get_mapdict_map().add_attr(new_obj, self.selector, w_value) - def read(self, obj, selector): - if selector == self.selector: - return obj._mapdict_read_storage(self.position) - return self.back.read(obj, selector) - - def write(self, obj, selector, w_value): - if selector == self.selector: - obj._mapdict_write_storage(self.position, w_value) - return True - return self.back.write(obj, selector, w_value) - def delete(self, obj, selector): if selector == self.selector: # ok, attribute is deleted @@ -219,10 +260,10 @@ class PlainAttribute(AbstractAttribute): self._copy_attr(obj, new_obj) return new_obj - def index(self, selector): + def _index(self, selector): if selector == self.selector: return self.position - return self.back.index(selector) + return self.back._index(selector) def copy(self, obj): new_obj = self.back.copy(obj) @@ -232,9 +273,6 @@ class PlainAttribute(AbstractAttribute): def length(self): return self.position + 1 - def get_terminator(self): - return self.back.get_terminator() - def set_terminator(self, obj, terminator): new_obj = self.back.set_terminator(obj, terminator) self._copy_attr(obj, new_obj) @@ -268,6 +306,24 @@ def _become(w_obj, new_obj): # RPython reasons w_obj._set_mapdict_storage_and_map(new_obj.storage, new_obj.map) +class IndexCache(object): + def __init__(self, space): + assert space.config.objspace.std.withmethodcache + SIZE = 1 << space.config.objspace.std.methodcachesizeexp + self.attrs = [None] * SIZE + self._empty_selector = (None, INVALID) + self.selectors = [self._empty_selector] * SIZE + self.indices = [0] * SIZE + if space.config.objspace.std.withmethodcachecounter: + self.hits = {} + self.misses = {} + + def clear(self): + for i in range(len(self.attrs)): + self.attrs[i] = None + for i in range(len(self.selectors)): + self.selectors[i] = self._empty_selector + # ____________________________________________________________ # object implementation @@ -328,7 +384,7 @@ class BaseMapdictObject: # slightly evil to make it inherit from W_Root assert flag def getclass(self, space): - return self._get_mapdict_map().w_cls + return self._get_mapdict_map().terminator.w_cls def setclass(self, space, w_cls): new_obj = self._get_mapdict_map().set_terminator(self, w_cls.terminator) @@ -373,6 +429,7 @@ class ObjectMixin(object): self.storage = make_sure_not_resized([None] * map.size_estimate()) def _mapdict_read_storage(self, index): + assert index >= 0 return self.storage[index] def _mapdict_write_storage(self, index, value): self.storage[index] = value @@ -440,6 +497,7 @@ def _make_subclass_size_n(supercls, n): return rerased.unerase_fixedsizelist(erased, W_Root) def _mapdict_read_storage(self, index): + assert index >= 0 for i in rangenmin1: if index == i: erased = getattr(self, "_value%s" % i) @@ -604,30 +662,54 @@ class CacheEntry(object): map = None version_tag = None index = 0 + w_method = None # for callmethod success_counter = 0 failure_counter = 0 + def is_valid_for_obj(self, w_obj): + map = w_obj._get_mapdict_map() + return self.is_valid_for_map(map) + + def is_valid_for_map(self, map): + if map is self.map: + version_tag = map.terminator.w_cls.version_tag() + if version_tag is self.version_tag: + # everything matches, it's incredibly fast + if map.space.config.objspace.std.withmethodcachecounter: + self.success_counter += 1 + return True + return False + INVALID_CACHE_ENTRY = CacheEntry() INVALID_CACHE_ENTRY.map = objectmodel.instantiate(AbstractAttribute) # different from any real map ^^^ -INVALID_CACHE_ENTRY.map.w_cls = None +INVALID_CACHE_ENTRY.map.terminator = None + def init_mapdict_cache(pycode): num_entries = len(pycode.co_names_w) pycode._mapdict_caches = [INVALID_CACHE_ENTRY] * num_entries +def _fill_cache(pycode, nameindex, map, version_tag, index, w_method=None): + entry = pycode._mapdict_caches[nameindex] + if entry is INVALID_CACHE_ENTRY: + entry = CacheEntry() + pycode._mapdict_caches[nameindex] = entry + entry.map = map + entry.version_tag = version_tag + entry.index = index + entry.w_method = w_method + if pycode.space.config.objspace.std.withmethodcachecounter: + entry.failure_counter += 1 + def LOAD_ATTR_caching(pycode, w_obj, nameindex): # this whole mess is to make the interpreter quite a bit faster; it's not # used if we_are_jitted(). entry = pycode._mapdict_caches[nameindex] map = w_obj._get_mapdict_map() - if map is entry.map: - version_tag = map.w_cls.version_tag() - if version_tag is entry.version_tag: - # everything matches, it's incredibly fast - if pycode.space.config.objspace.std.withmethodcachecounter: - entry.success_counter += 1 - return w_obj._mapdict_read_storage(entry.index) + if entry.is_valid_for_map(map) and entry.w_method is None: + # everything matches, it's incredibly fast + return w_obj._mapdict_read_storage(entry.index) return LOAD_ATTR_slowpath(pycode, w_obj, nameindex, map) LOAD_ATTR_caching._always_inline_ = True @@ -635,7 +717,7 @@ def LOAD_ATTR_slowpath(pycode, w_obj, nameindex, map): space = pycode.space w_name = pycode.co_names_w[nameindex] if map is not None: - w_type = map.w_cls + w_type = map.terminator.w_cls w_descr = w_type.getattribute_if_not_from_object() if w_descr is not None: return space._handle_getattribute(w_descr, w_obj, w_name) @@ -655,17 +737,30 @@ def LOAD_ATTR_slowpath(pycode, w_obj, nameindex, map): if selector[1] != INVALID: index = map.index(selector) if index >= 0: - entry = pycode._mapdict_caches[nameindex] - if entry is INVALID_CACHE_ENTRY: - entry = CacheEntry() - pycode._mapdict_caches[nameindex] = entry - entry.map = map - entry.version_tag = version_tag - entry.index = index - if space.config.objspace.std.withmethodcachecounter: - entry.failure_counter += 1 + _fill_cache(pycode, nameindex, map, version_tag, index) return w_obj._mapdict_read_storage(index) if space.config.objspace.std.withmethodcachecounter: INVALID_CACHE_ENTRY.failure_counter += 1 return space.getattr(w_obj, w_name) LOAD_ATTR_slowpath._dont_inline_ = True + +def LOOKUP_METHOD_mapdict(f, nameindex, w_obj): + space = f.space + pycode = f.getcode() + entry = pycode._mapdict_caches[nameindex] + if entry.is_valid_for_obj(w_obj): + w_method = entry.w_method + if w_method is not None: + f.pushvalue(w_method) + f.pushvalue(w_obj) + return True + return False + +def LOOKUP_METHOD_mapdict_fill_cache_method(pycode, nameindex, w_obj, w_type, w_method): + version_tag = w_type.version_tag() + if version_tag is None: + return + map = w_obj._get_mapdict_map() + if map is None: + return + _fill_cache(pycode, nameindex, map, version_tag, -1, w_method) diff --git a/pypy/objspace/std/stringobject.py b/pypy/objspace/std/stringobject.py index b1f889c99d..ddb4eff97e 100644 --- a/pypy/objspace/std/stringobject.py +++ b/pypy/objspace/std/stringobject.py @@ -937,10 +937,17 @@ def str_translate__String_ANY_ANY(space, w_string, w_table, w_deletechars=''): string = w_string._value chars = [] - for char in string: - w_char = W_StringObject.PREBUILT[ord(char)] - if not space.is_true(space.contains(w_deletechars, w_char)): - chars.append(table[ord(char)]) + deletechars = space.str_w(w_deletechars) + if len(deletechars) == 0: + for char in string: + chars.append(table[ord(char)]) + else: + deletion_table = [False] * 256 + for c in deletechars: + deletion_table[ord(c)] = True + for char in string: + if not deletion_table[ord(char)]: + chars.append(table[ord(char)]) return W_StringObject(''.join(chars)) def str_decode__String_ANY_ANY(space, w_string, w_encoding=None, w_errors=None): diff --git a/pypy/objspace/std/test/test_dictmultiobject.py b/pypy/objspace/std/test/test_dictmultiobject.py index 8a052b46a4..dc10ab9c24 100644 --- a/pypy/objspace/std/test/test_dictmultiobject.py +++ b/pypy/objspace/std/test/test_dictmultiobject.py @@ -616,6 +616,7 @@ class Config: withdictmeasurement = False withsmalldicts = False withcelldict = False + withmethodcache = False class opcodes: CALL_LIKELY_BUILTIN = False diff --git a/pypy/objspace/std/test/test_mapdict.py b/pypy/objspace/std/test/test_mapdict.py index baf857cd03..5df7b5cc91 100644 --- a/pypy/objspace/std/test/test_mapdict.py +++ b/pypy/objspace/std/test/test_mapdict.py @@ -24,13 +24,13 @@ class Object(Object): hasdict = False def test_plain_attribute(): - space = " " w_cls = "class" aa = PlainAttribute(("b", DICT), PlainAttribute(("a", DICT), Terminator(space, w_cls))) assert aa.space is space - assert aa.w_cls is w_cls + assert aa.terminator.w_cls is w_cls + assert aa.get_terminator() is aa.terminator obj = Object() obj.map, obj.storage = aa, [10, 20] @@ -604,7 +604,8 @@ class AppTestWithMapDictAndCounters(object): from pypy.interpreter import gateway cls.space = gettestobjspace( **{"objspace.std.withmapdict": True, - "objspace.std.withmethodcachecounter": True}) + "objspace.std.withmethodcachecounter": True, + "objspace.opcodes.CALL_METHOD": True}) # def check(space, w_func, name): w_code = space.getattr(w_func, space.wrap('func_code')) @@ -785,6 +786,135 @@ class AppTestWithMapDictAndCounters(object): res = self.check(f, 'x') assert res == (0, 0, 1) + def test_call_method_uses_cache(self): + # bit sucky + global C + + class C(object): + def m(*args): + return args + C.sm = staticmethod(C.m.im_func) + C.cm = classmethod(C.m.im_func) + + exec """if 1: + + def f(): + c = C() + res = c.m(1) + assert res == (c, 1) + return 42 + + def g(): + c = C() + res = c.sm(1) + assert res == (1, ) + return 42 + + def h(): + c = C() + res = c.cm(1) + assert res == (C, 1) + return 42 + """ + res = self.check(f, 'm') + assert res == (1, 0, 0) + res = self.check(f, 'm') + assert res == (0, 1, 0) + res = self.check(f, 'm') + assert res == (0, 1, 0) + res = self.check(f, 'm') + assert res == (0, 1, 0) + + # static methods are not cached + res = self.check(g, 'sm') + assert res == (0, 0, 0) + res = self.check(g, 'sm') + assert res == (0, 0, 0) + + # neither are class methods + res = self.check(h, 'cm') + assert res == (0, 0, 0) + res = self.check(h, 'cm') + assert res == (0, 0, 0) + + def test_mix_cache_bug(self): + # bit sucky + global C + + class C(object): + def m(*args): + return args + + exec """if 1: + + def f(): + c = C() + res = c.m(1) + assert res == (c, 1) + bm = c.m + res = bm(1) + assert res == (c, 1) + return 42 + + """ + res = self.check(f, 'm') + assert res == (1, 1, 1) + res = self.check(f, 'm') + assert res == (0, 2, 1) + res = self.check(f, 'm') + assert res == (0, 2, 1) + res = self.check(f, 'm') + assert res == (0, 2, 1) + +class AppTestGlobalCaching(AppTestWithMapDict): + def setup_class(cls): + cls.space = gettestobjspace( + **{"objspace.std.withmethodcachecounter": True, + "objspace.std.withmapdict": True, + "objspace.opcodes.CALL_METHOD": True}) + + def test_mix_classes(self): + import __pypy__ + class A(object): + def f(self): + return 42 + class B(object): + def f(self): + return 43 + class C(object): + def f(self): + return 44 + l = [A(), B(), C()] * 10 + __pypy__.reset_method_cache_counter() + # 'exec' to make sure that a.f() is compiled with CALL_METHOD + exec """for i, a in enumerate(l): + assert a.f() == 42 + i % 3 +""" + cache_counter = __pypy__.mapdict_cache_counter("f") + assert cache_counter[0] >= 15 + assert cache_counter[1] >= 3 # should be (27, 3) + assert sum(cache_counter) == 30 + + def test_mix_classes_attribute(self): + import __pypy__ + class A(object): + def __init__(self): + self.x = 42 + class B(object): + def __init__(self): + self.x = 43 + class C(object): + def __init__(self): + self.x = 44 + l = [A(), B(), C()] * 10 + __pypy__.reset_method_cache_counter() + for i, a in enumerate(l): + assert a.x == 42 + i % 3 + cache_counter = __pypy__.mapdict_cache_counter("x") + assert cache_counter[0] >= 15 + assert cache_counter[1] >= 3 # should be (27, 3) + assert sum(cache_counter) == 30 + class TestDictSubclassShortcutBug(object): def setup_class(cls): cls.space = gettestobjspace( diff --git a/pypy/rlib/clibffi.py b/pypy/rlib/clibffi.py index 6414be6995..5443b9a664 100644 --- a/pypy/rlib/clibffi.py +++ b/pypy/rlib/clibffi.py @@ -432,7 +432,8 @@ class CallbackFuncPtr(AbstractFuncPtr): flags=FUNCFLAG_CDECL): AbstractFuncPtr.__init__(self, "callback", argtypes, restype, flags) self.ll_closure = closureHeap.alloc() - self.ll_userdata = lltype.malloc(USERDATA_P.TO, flavor='raw') + self.ll_userdata = lltype.malloc(USERDATA_P.TO, flavor='raw', + track_allocation=False) self.ll_userdata.callback = rffi.llhelper(CALLBACK_TP, func) self.ll_userdata.addarg = additional_arg res = c_ffi_prep_closure(self.ll_closure, self.ll_cif, @@ -447,7 +448,7 @@ class CallbackFuncPtr(AbstractFuncPtr): closureHeap.free(self.ll_closure) self.ll_closure = lltype.nullptr(FFI_CLOSUREP.TO) if self.ll_userdata: - lltype.free(self.ll_userdata, flavor='raw') + lltype.free(self.ll_userdata, flavor='raw', track_allocation=False) self.ll_userdata = lltype.nullptr(USERDATA_P.TO) class RawFuncPtr(AbstractFuncPtr): diff --git a/pypy/rlib/jit.py b/pypy/rlib/jit.py index 04326afc04..884c2d7387 100644 --- a/pypy/rlib/jit.py +++ b/pypy/rlib/jit.py @@ -4,6 +4,7 @@ from pypy.rpython.extregistry import ExtRegistryEntry from pypy.rlib.objectmodel import CDefinedIntSymbolic from pypy.rlib.objectmodel import keepalive_until_here from pypy.rlib.unroll import unrolling_iterable +from pypy.rlib.nonconst import NonConstant def purefunction(func): """ Decorate a function as pure. Pure means precisely that: @@ -145,6 +146,14 @@ class Entry(ExtRegistryEntry): return hop.inputconst(lltype.Signed, _we_are_jitted) +def current_trace_length(): + """During JIT tracing, returns the current trace length (as a constant). + If not tracing, returns -1.""" + if NonConstant(False): + return 73 + return -1 +current_trace_length.oopspec = 'jit.current_trace_length()' + def jit_debug(string, arg1=-sys.maxint-1, arg2=-sys.maxint-1, arg3=-sys.maxint-1, arg4=-sys.maxint-1): """When JITted, cause an extra operation DEBUG_MERGE_POINT to appear in diff --git a/pypy/rlib/rarithmetic.py b/pypy/rlib/rarithmetic.py index e552b385c9..84552eaf59 100644 --- a/pypy/rlib/rarithmetic.py +++ b/pypy/rlib/rarithmetic.py @@ -501,6 +501,11 @@ class r_singlefloat(object): def __cmp__(self, other): raise TypeError("not supported on r_singlefloat instances") + def __eq__(self, other): + return self.__class__ is other.__class__ and self._bytes == other._bytes + + def __ne__(self, other): + return not self.__eq__(other) class For_r_singlefloat_values_Entry(extregistry.ExtRegistryEntry): _type_ = r_singlefloat diff --git a/pypy/rlib/rgc.py b/pypy/rlib/rgc.py index 96dc17078f..4fc921c3e6 100644 --- a/pypy/rlib/rgc.py +++ b/pypy/rlib/rgc.py @@ -379,6 +379,11 @@ def dump_rpy_heap(fd): "NOT_RPYTHON" raise NotImplementedError +def get_typeids_z(): + "NOT_RPYTHON" + raise NotImplementedError + +ARRAY_OF_CHAR = lltype.Array(lltype.Char) NULL_GCREF = lltype.nullptr(llmemory.GCREF.TO) class _GcRef(object): @@ -530,3 +535,12 @@ class Entry(ExtRegistryEntry): vlist = hop.inputargs(lltype.Signed) hop.exception_is_here() return hop.genop('gc_dump_rpy_heap', vlist, resulttype = hop.r_result) + +class Entry(ExtRegistryEntry): + _about_ = get_typeids_z + def compute_result_annotation(self): + from pypy.annotation.model import SomePtr + return SomePtr(lltype.Ptr(ARRAY_OF_CHAR)) + def specialize_call(self, hop): + hop.exception_is_here() + return hop.genop('gc_typeids_z', [], resulttype = hop.r_result) diff --git a/pypy/rlib/streamio.py b/pypy/rlib/streamio.py index 16f05ed89e..cfcbb96092 100644 --- a/pypy/rlib/streamio.py +++ b/pypy/rlib/streamio.py @@ -12,7 +12,7 @@ Based on sio.py from Guido van Rossum. * some other methods also have no default parameters. * close() should be called exactly once and no further operations performed; there is no __del__() closing the stream for you. - * some methods may raise NotImplementedError. + * some methods may raise MyNotImplementedError. * peek() returns some (or no) characters that have already been read ahead. * flushable() returns True/False if flushing that stream is useful/pointless. @@ -54,6 +54,12 @@ OS_MODE = {('r', False): O_RDONLY, ('a', True): O_RDWR | O_CREAT, } +class MyNotImplementedError(Exception): + """ + Catching NotImplementedError is not RPython, so we use this custom class + instead of it + """ + # ____________________________________________________________ @@ -209,16 +215,16 @@ class Stream(object): some methods.""" def read(self, n): - raise NotImplementedError + raise MyNotImplementedError def write(self, data): - raise NotImplementedError + raise MyNotImplementedError def tell(self): - raise NotImplementedError + raise MyNotImplementedError def seek(self, offset, whence): - raise NotImplementedError + raise MyNotImplementedError def readall(self): bufsize = 8192 @@ -251,7 +257,7 @@ class Stream(object): return ''.join(result) def truncate(self, size): - raise NotImplementedError + raise MyNotImplementedError def flush_buffers(self): pass @@ -487,7 +493,7 @@ class BufferingInputStream(Stream): if self.lines or self.buf: try: self.do_seek(self.tell(), 0) - except NotImplementedError: + except MyNotImplementedError: pass else: self.lines = [] @@ -535,14 +541,14 @@ class BufferingInputStream(Stream): self.buf = "" try: self.do_seek(offset, 1) - except NotImplementedError: + except MyNotImplementedError: intoffset = offset2int(offset) self.read(intoffset) return if whence == 2: try: self.do_seek(offset, 2) - except NotImplementedError: + except MyNotImplementedError: pass else: self.lines = [] @@ -1019,7 +1025,7 @@ class TextInputFilter(Stream): if self.buf: try: self.base.seek(-len(self.buf), 1) - except NotImplementedError: + except MyNotImplementedError: pass else: self.buf = "" diff --git a/pypy/rlib/test/test_rarithmetic.py b/pypy/rlib/test/test_rarithmetic.py index 95445b7398..19218ab808 100644 --- a/pypy/rlib/test/test_rarithmetic.py +++ b/pypy/rlib/test/test_rarithmetic.py @@ -325,6 +325,15 @@ def test_r_singlefloat(): assert float(x) != 2.1 assert abs(float(x) - 2.1) < 1E-6 +def test_r_singlefloat_eq(): + x = r_singlefloat(2.5) # exact number + y = r_singlefloat(2.5) + assert x == y + assert not x != y + assert not x == 2.5 + assert x != 2.5 + py.test.raises(TypeError, "x>y") + class BaseTestRarithmetic(BaseRtypingTest): def test_formatd(self): from pypy.rlib.rarithmetic import formatd @@ -358,7 +367,17 @@ class BaseTestRarithmetic(BaseRtypingTest): assert res == 1.0 res = self.interpret(f, [1]) - assert res == 1e-100 + assert res == 1e-100 + + def test_compare_singlefloat_crashes(self): + from pypy.rlib.rarithmetic import r_singlefloat + from pypy.rpython.error import MissingRTypeOperation + def f(x): + a = r_singlefloat(x) + b = r_singlefloat(x+1) + return a == b + py.test.raises(MissingRTypeOperation, "self.interpret(f, [42.0])") + class TestLLtype(BaseTestRarithmetic, LLRtypeMixin): pass diff --git a/pypy/rpython/annlowlevel.py b/pypy/rpython/annlowlevel.py index d0ca5da0f1..f1b2f9b9dc 100644 --- a/pypy/rpython/annlowlevel.py +++ b/pypy/rpython/annlowlevel.py @@ -136,7 +136,7 @@ class MixLevelAnnotatorPolicy(LowLevelAnnotatorPolicy): return funcdesc.cachedgraph(TYPE, alt_name=valid_identifier(alt_name)) -class MixLevelHelperAnnotator: +class MixLevelHelperAnnotator(object): def __init__(self, rtyper): self.rtyper = rtyper diff --git a/pypy/rpython/llinterp.py b/pypy/rpython/llinterp.py index fab8728d44..4a725cbcd1 100644 --- a/pypy/rpython/llinterp.py +++ b/pypy/rpython/llinterp.py @@ -912,6 +912,9 @@ class LLFrame(object): def op_gc_dump_rpy_heap(self): raise NotImplementedError("gc_dump_rpy_heap") + def op_gc_typeids_z(self): + raise NotImplementedError("gc_typeids_z") + def op_do_malloc_fixedsize_clear(self): raise NotImplementedError("do_malloc_fixedsize_clear") diff --git a/pypy/rpython/lltypesystem/llarena.py b/pypy/rpython/lltypesystem/llarena.py index 0f2084b8e4..6377f6c00e 100644 --- a/pypy/rpython/lltypesystem/llarena.py +++ b/pypy/rpython/lltypesystem/llarena.py @@ -440,7 +440,8 @@ elif os.name == 'posix': [rffi.INT], rffi.INT, sandboxsafe=True, _nowrapper=True) - _dev_zero = rffi.str2charp_immortal('/dev/zero') # prebuilt + _dev_zero = rffi.str2charp('/dev/zero') # prebuilt + lltype.render_immortal(_dev_zero) def clear_large_memory_chunk(baseaddr, size): # on some Unixy platforms, reading from /dev/zero is the fastest way diff --git a/pypy/rpython/lltypesystem/llmemory.py b/pypy/rpython/lltypesystem/llmemory.py index 11d7533ccf..48211ee8d2 100644 --- a/pypy/rpython/lltypesystem/llmemory.py +++ b/pypy/rpython/lltypesystem/llmemory.py @@ -766,8 +766,10 @@ class _gctransformed_wref(lltype._container): return '<%s>' % (self,) def __str__(self): return 'gctransformed_wref(%s)' % (self._ptr,) - def _normalizedcontainer(self): - return self._ptr._obj + def _normalizedcontainer(self, check=True): + return self._ptr._getobj(check=check)._normalizedcontainer(check=check) + def _was_freed(self): + return self._ptr._was_freed() # ____________________________________________________________ diff --git a/pypy/rpython/lltypesystem/lloperation.py b/pypy/rpython/lltypesystem/lloperation.py index ab4fbe005f..b723e6e975 100644 --- a/pypy/rpython/lltypesystem/lloperation.py +++ b/pypy/rpython/lltypesystem/lloperation.py @@ -476,6 +476,7 @@ LL_OPERATIONS = { 'gc_get_rpy_type_index': LLOp(), 'gc_is_rpy_instance' : LLOp(), 'gc_dump_rpy_heap' : LLOp(), + 'gc_typeids_z' : LLOp(), # ------- JIT & GC interaction, only for some GCs ---------- diff --git a/pypy/rpython/lltypesystem/lltype.py b/pypy/rpython/lltypesystem/lltype.py index ff94498e07..2885a7ed7a 100644 --- a/pypy/rpython/lltypesystem/lltype.py +++ b/pypy/rpython/lltypesystem/lltype.py @@ -1868,6 +1868,13 @@ def free(p, flavor, track_allocation=True): leakfinder.remember_free(p._obj0) p._obj0._free() +def render_immortal(p, track_allocation=True): + T = typeOf(p) + if not isinstance(T, Ptr) or p._togckind() != 'raw': + raise TypeError, "free(): only for pointers to non-gc containers" + if track_allocation: + leakfinder.remember_free(p._obj0) + def _make_scoped_allocator(T): class ScopedAlloc: def __init__(self, n=None, zero=False): diff --git a/pypy/rpython/lltypesystem/rffi.py b/pypy/rpython/lltypesystem/rffi.py index a52b8c85fd..7f15a13dc8 100644 --- a/pypy/rpython/lltypesystem/rffi.py +++ b/pypy/rpython/lltypesystem/rffi.py @@ -607,15 +607,6 @@ def make_string_mappings(strtype): return array str2charp._annenforceargs_ = [strtype] - def str2charp_immortal(s): - "NOT_RPYTHON" - array = lltype.malloc(TYPEP.TO, len(s) + 1, flavor='raw', - immortal=True) - for i in range(len(s)): - array[i] = s[i] - array[len(s)] = lastchar - return array - def free_charp(cp): lltype.free(cp, flavor='raw') @@ -734,19 +725,19 @@ def make_string_mappings(strtype): l = [cp[i] for i in range(size)] return emptystr.join(l) - return (str2charp, str2charp_immortal, free_charp, charp2str, + return (str2charp, free_charp, charp2str, get_nonmovingbuffer, free_nonmovingbuffer, alloc_buffer, str_from_buffer, keep_buffer_alive_until_here, charp2strn, charpsize2str, ) -(str2charp, str2charp_immortal, free_charp, charp2str, +(str2charp, free_charp, charp2str, get_nonmovingbuffer, free_nonmovingbuffer, alloc_buffer, str_from_buffer, keep_buffer_alive_until_here, charp2strn, charpsize2str, ) = make_string_mappings(str) -(unicode2wcharp, unicode2wcharp_immortal, free_wcharp, wcharp2unicode, +(unicode2wcharp, free_wcharp, wcharp2unicode, get_nonmoving_unicodebuffer, free_nonmoving_unicodebuffer, alloc_unicodebuffer, unicode_from_buffer, keep_unicodebuffer_alive_until_here, wcharp2unicoden, wcharpsize2unicode, diff --git a/pypy/rpython/memory/gc/base.py b/pypy/rpython/memory/gc/base.py index 6321450b4a..593cc90d16 100644 --- a/pypy/rpython/memory/gc/base.py +++ b/pypy/rpython/memory/gc/base.py @@ -247,6 +247,21 @@ class GCBase(object): (not self.config.taggedpointers or llmemory.cast_adr_to_int(addr) & 1 == 0)) + def enumerate_all_roots(self, callback, arg): + """For each root object, invoke callback(obj, arg). + 'callback' should not be a bound method. + Note that this method is not suitable for actually doing the + collection in a moving GC, because you cannot write back a + modified address. It is there only for inspection. + """ + # overridden in some subclasses, for GCs which have an additional + # list of last generation roots + callback2, attrname = _convert_callback_formats(callback) # :-/ + setattr(self, attrname, arg) + self.root_walker.walk_roots(callback2, callback2, callback2) + self.run_finalizers.foreach(callback, arg) + enumerate_all_roots._annspecialcase_ = 'specialize:arg(1)' + def debug_check_consistency(self): """To use after a collection. If self.DEBUG is set, this enumerates all roots and traces all objects to check if we didn't @@ -260,8 +275,7 @@ class GCBase(object): self._debug_pending = self.AddressStack() if not we_are_translated(): self.root_walker._walk_prebuilt_gc(self._debug_record) - callback = GCBase._debug_callback - self.root_walker.walk_roots(callback, callback, callback) + self.enumerate_all_roots(GCBase._debug_callback, self) pending = self._debug_pending while pending.non_empty(): obj = pending.pop() @@ -275,9 +289,8 @@ class GCBase(object): seen.add(obj) self.debug_check_object(obj) self._debug_pending.append(obj) - def _debug_callback(self, root): - obj = root.address[0] - ll_assert(bool(obj), "NULL address from walk_roots()") + @staticmethod + def _debug_callback(obj, self): self._debug_record(obj) def _debug_callback2(self, pointer, ignored): obj = pointer.address[0] @@ -432,3 +445,17 @@ def read_float_from_env(varname): if factor != 1: return 0.0 return value + +def _convert_callback_formats(callback): + callback = getattr(callback, 'im_func', callback) + if callback not in _converted_callback_formats: + def callback2(gc, root): + obj = root.address[0] + ll_assert(bool(obj), "NULL address from walk_roots()") + callback(obj, getattr(gc, attrname)) + attrname = '_callback2_arg%d' % len(_converted_callback_formats) + _converted_callback_formats[callback] = callback2, attrname + return _converted_callback_formats[callback] + +_convert_callback_formats._annspecialcase_ = 'specialize:memo' +_converted_callback_formats = {} diff --git a/pypy/rpython/memory/gc/generation.py b/pypy/rpython/memory/gc/generation.py index 69426f7650..1f35127375 100644 --- a/pypy/rpython/memory/gc/generation.py +++ b/pypy/rpython/memory/gc/generation.py @@ -572,16 +572,10 @@ class GenerationGC(SemiSpaceGC): def _compute_current_nursery_hash(self, obj): return intmask(llmemory.cast_adr_to_int(obj) + self.nursery_hash_base) - def heap_stats_walk_roots(self): - self.last_generation_root_objects.foreach( - self._track_heap_ext, None) - self.root_walker.walk_roots( - SemiSpaceGC._track_heap_root, - SemiSpaceGC._track_heap_root, - SemiSpaceGC._track_heap_root) - - def _track_heap_ext(self, adr, ignored): - self.trace(adr, self.track_heap_parent, adr) + def enumerate_all_roots(self, callback, arg): + self.last_generation_root_objects.foreach(callback, arg) + SemiSpaceGC.enumerate_all_roots(self, callback, arg) + enumerate_all_roots._annspecialcase_ = 'specialize:arg(1)' def debug_check_object(self, obj): """Check the invariants about 'obj' that should be true diff --git a/pypy/rpython/memory/gc/inspector.py b/pypy/rpython/memory/gc/inspector.py index 607f8a2775..cd665144d4 100644 --- a/pypy/rpython/memory/gc/inspector.py +++ b/pypy/rpython/memory/gc/inspector.py @@ -4,25 +4,22 @@ Utility RPython functions to inspect objects in the GC. from pypy.rpython.lltypesystem import lltype, llmemory, rffi from pypy.rlib.objectmodel import free_non_gc_object from pypy.rpython.module.ll_os import underscore_on_windows -from pypy.rlib import rposix +from pypy.rlib import rposix, rgc from pypy.rpython.memory.support import AddressDict, get_address_stack # ---------- implementation of pypy.rlib.rgc.get_rpy_roots() ---------- -def _counting_rpy_root(gc, root): +def _counting_rpy_root(obj, gc): gc._count_rpy += 1 def _do_count_rpy_roots(gc): gc._count_rpy = 0 - gc.root_walker.walk_roots( - _counting_rpy_root, - _counting_rpy_root, - _counting_rpy_root) + gc.enumerate_all_roots(_counting_rpy_root, gc) return gc._count_rpy -def _append_rpy_root(gc, root): +def _append_rpy_root(obj, gc): # Can use the gc list, but should not allocate! # It is essential that the list is not resizable! lst = gc._list_rpy @@ -30,15 +27,12 @@ def _append_rpy_root(gc, root): if index >= len(lst): raise ValueError gc._count_rpy = index + 1 - lst[index] = llmemory.cast_adr_to_ptr(root.address[0], llmemory.GCREF) + lst[index] = llmemory.cast_adr_to_ptr(obj, llmemory.GCREF) def _do_append_rpy_roots(gc, lst): gc._count_rpy = 0 gc._list_rpy = lst - gc.root_walker.walk_roots( - _append_rpy_root, - _append_rpy_root, - _append_rpy_root) + gc.enumerate_all_roots(_append_rpy_root, gc) gc._list_rpy = None def get_rpy_roots(gc): @@ -172,12 +166,7 @@ class HeapDumper: self.pending.append(obj) def add_roots(self): - self.gc._heap_dumper = self - self.gc.root_walker.walk_roots( - _hd_add_root, - _hd_add_root, - _hd_add_root) - self.gc._heap_dumper = None + self.gc.enumerate_all_roots(_hd_add_root, self) pendingroots = self.pending self.pending = AddressStack() self.walk(pendingroots) @@ -188,8 +177,8 @@ class HeapDumper: while pending.non_empty(): self.writeobj(pending.pop()) -def _hd_add_root(gc, root): - gc._heap_dumper.add(root.address[0]) +def _hd_add_root(obj, heap_dumper): + heap_dumper.add(obj) def dump_rpy_heap(gc, fd): heapdumper = HeapDumper(gc, fd) @@ -198,3 +187,7 @@ def dump_rpy_heap(gc, fd): heapdumper.flush() heapdumper.delete() return True + +def get_typeids_z(gc): + srcaddress = gc.root_walker.gcdata.typeids_z + return llmemory.cast_adr_to_ptr(srcaddress, lltype.Ptr(rgc.ARRAY_OF_CHAR)) diff --git a/pypy/rpython/memory/gc/minimark.py b/pypy/rpython/memory/gc/minimark.py index d6045d6349..9dffd5cdaa 100644 --- a/pypy/rpython/memory/gc/minimark.py +++ b/pypy/rpython/memory/gc/minimark.py @@ -1,3 +1,36 @@ +""" MiniMark GC. + +Environment variables can be used to fine-tune the following parameters: + + PYPY_GC_NURSERY The nursery size. Defaults to half the size of + the L2 cache. Try values like '1.2MB'. + + PYPY_GC_MAJOR_COLLECT Major collection memory factor. Default is '1.82', + which means trigger a major collection when the + memory consumed equals 1.82 times the memory + really used at the end of the previous major + collection. + + PYPY_GC_GROWTH Major collection threshold's max growth rate. + Default is '1.3'. Useful to collect more often + than normally on sudden memory growth, e.g. when + there is a temporary peak in memory usage. + + PYPY_GC_MAX The max heap size. If coming near this limit, it + will first collect more often, then raise an + RPython MemoryError, and if that is not enough, + crash the program with a fatal error. Try values + like '1.6GB'. + + PYPY_GC_MIN Don't collect while the memory size is below this + limit. Useful to avoid spending all the time in + the GC in very small programs. Defaults to 8 + times the nursery. +""" +# XXX Should find a way to bound the major collection threshold by the +# XXX total addressable size. Maybe by keeping some minimarkpage arenas +# XXX pre-reserved, enough for a few nursery collections? What about +# XXX raw-malloced memory? import sys from pypy.rpython.lltypesystem import lltype, llmemory, llarena, llgroup from pypy.rpython.lltypesystem.lloperation import llop @@ -77,7 +110,7 @@ class MiniMarkGC(MovingGCBase): # During a minor collection, the objects in the nursery that are # moved outside are changed in-place: their header is replaced with - # the value -1, and the following word is set to the address of + # the value -42, and the following word is set to the address of # where the object was moved. This means that all objects in the # nursery need to be at least 2 words long, but objects outside the # nursery don't need to. @@ -87,13 +120,8 @@ class MiniMarkGC(MovingGCBase): TRANSLATION_PARAMS = { # Automatically adjust the size of the nursery and the - # 'major_collection_threshold' from the environment. For - # 'nursery_size' it will look it up in the env var - # PYPY_GC_NURSERY and fall back to half the size of - # the L2 cache. For 'major_collection_threshold' it will look - # it up in the env var PYPY_GC_MAJOR_COLLECT. It also sets - # 'max_heap_size' to PYPY_GC_MAX. Finally, PYPY_GC_MIN sets - # the minimal value of 'next_major_collection_threshold'. + # 'major_collection_threshold' from the environment. + # See docstring at the start of the file. "read_from_env": True, # The size of the nursery. Note that this is only used as a @@ -121,6 +149,13 @@ class MiniMarkGC(MovingGCBase): # we trigger the next major collection. "major_collection_threshold": 1.82, + # Threshold to avoid that the total heap size grows by a factor of + # major_collection_threshold at every collection: it can only + # grow at most by the following factor from one collection to the + # next. Used e.g. when there is a sudden, temporary peak in memory + # usage; this avoids that the upper bound grows too fast. + "growth_rate_max": 1.3, + # The number of array indices that are mapped to a single bit in # write_barrier_from_array(). Must be a power of two. The default # value of 128 means that card pages are 512 bytes (1024 on 64-bits) @@ -146,6 +181,7 @@ class MiniMarkGC(MovingGCBase): arena_size=64*WORD, small_request_threshold=5*WORD, major_collection_threshold=2.5, + growth_rate_max=2.5, # for tests card_page_indices=0, large_object=8*WORD, large_object_gcptrs=10*WORD, @@ -157,6 +193,7 @@ class MiniMarkGC(MovingGCBase): self.nursery_size = nursery_size self.small_request_threshold = small_request_threshold self.major_collection_threshold = major_collection_threshold + self.growth_rate_max = growth_rate_max self.num_major_collects = 0 self.min_heap_size = 0.0 self.max_heap_size = 0.0 @@ -178,6 +215,7 @@ class MiniMarkGC(MovingGCBase): self.nursery = NULL self.nursery_free = NULL self.nursery_top = NULL + self.debug_always_do_minor_collect = False # # The ArenaCollection() handles the nonmovable objects allocation. if ArenaCollectionClass is None: @@ -245,6 +283,10 @@ class MiniMarkGC(MovingGCBase): # From there on, the GC is fully initialized and the code # below can use it newsize = base.read_from_env('PYPY_GC_NURSERY') + # PYPY_GC_NURSERY=1 forces a minor collect for every malloc. + # Useful to debug external factors, like trackgcroot or the + # handling of the write barrier. + self.debug_always_do_minor_collect = newsize == 1 if newsize <= 0: newsize = generation.estimate_best_nursery_size() if newsize <= 0: @@ -252,9 +294,13 @@ class MiniMarkGC(MovingGCBase): newsize = max(newsize, minsize) # major_coll = base.read_float_from_env('PYPY_GC_MAJOR_COLLECT') - if major_coll >= 1.0: + if major_coll > 1.0: self.major_collection_threshold = major_coll # + growth = base.read_float_from_env('PYPY_GC_GROWTH') + if growth > 1.0: + self.growth_rate_max = growth + # min_heap_size = base.read_uint_from_env('PYPY_GC_MIN') if min_heap_size > 0: self.min_heap_size = float(min_heap_size) @@ -290,11 +336,19 @@ class MiniMarkGC(MovingGCBase): # initialize the threshold self.min_heap_size = max(self.min_heap_size, self.nursery_size * self.major_collection_threshold) + self.next_major_collection_threshold = self.min_heap_size self.set_major_threshold_from(0.0) debug_stop("gc-set-nursery-size") - def set_major_threshold_from(self, threshold): + + def set_major_threshold_from(self, threshold, reserving_size=0): # Set the next_major_collection_threshold. + threshold_max = (self.next_major_collection_threshold * + self.growth_rate_max) + if threshold > threshold_max: + threshold = threshold_max + # + threshold += reserving_size if threshold < self.min_heap_size: threshold = self.min_heap_size # @@ -444,6 +498,10 @@ class MiniMarkGC(MovingGCBase): result = self.nursery_free self.nursery_free = result + totalsize ll_assert(self.nursery_free <= self.nursery_top, "nursery overflow") + # + if self.debug_always_do_minor_collect: + self.nursery_free = self.nursery_top + # return result collect_and_reserve._dont_inline_ = True @@ -647,10 +705,13 @@ class MiniMarkGC(MovingGCBase): def is_forwarded(self, obj): """Returns True if the nursery obj is marked as forwarded. Implemented a bit obscurely by checking an unrelated flag - that can never be set on a young object -- except if tid == -1. + that can never be set on a young object -- except if tid == -42. """ assert self.is_in_nursery(obj) - return self.header(obj).tid & GCFLAG_FINALIZATION_ORDERING + result = (self.header(obj).tid & GCFLAG_FINALIZATION_ORDERING != 0) + if result: + ll_assert(self.header(obj).tid == -42, "bogus header for young obj") + return result def get_forwarding_address(self, obj): return llmemory.cast_adr_to_ptr(obj, FORWARDSTUBPTR).forw @@ -734,6 +795,10 @@ class MiniMarkGC(MovingGCBase): def JIT_max_size_of_young_obj(cls): return cls.TRANSLATION_PARAMS['large_object'] + @classmethod + def JIT_minimal_size_in_nursery(cls): + return cls.minimal_size_in_nursery + def write_barrier(self, newvalue, addr_struct): if self.header(addr_struct).tid & GCFLAG_NO_YOUNG_PTRS: self.remember_young_pointer(addr_struct, newvalue) @@ -906,7 +971,7 @@ class MiniMarkGC(MovingGCBase): # # Now all live nursery objects should be out. Update the # young weakrefs' targets. - if self.young_objects_with_weakrefs.length() > 0: + if self.young_objects_with_weakrefs.non_empty(): self.invalidate_young_weakrefs() # # Clear this mapping. @@ -1000,7 +1065,7 @@ class MiniMarkGC(MovingGCBase): obj = oldlist.pop() # # Add the flag GCFLAG_NO_YOUNG_PTRS. All live objects should have - # this flag after a nursery collection. + # this flag set after a nursery collection. self.header(obj).tid |= GCFLAG_NO_YOUNG_PTRS # # Trace the 'obj' to replace pointers to nursery with pointers @@ -1063,7 +1128,7 @@ class MiniMarkGC(MovingGCBase): # nursery are kept unchanged in this step. llmemory.raw_memcopy(obj - size_gc_header, newhdr, totalsize) # - # Set the old object's tid to -1 (containing all flags) and + # Set the old object's tid to -42 (containing all flags) and # replace the old object's content with the target address. # A bit of no-ops to convince llarena that we are changing # the layout, in non-translated versions. @@ -1071,7 +1136,7 @@ class MiniMarkGC(MovingGCBase): llarena.arena_reset(obj - size_gc_header, totalsize, 0) llarena.arena_reserve(obj - size_gc_header, size_gc_header + llmemory.sizeof(FORWARDSTUB)) - self.header(obj).tid = -1 + self.header(obj).tid = -42 newobj = newhdr + size_gc_header llmemory.cast_adr_to_ptr(obj, FORWARDSTUBPTR).forw = newobj # @@ -1140,6 +1205,10 @@ class MiniMarkGC(MovingGCBase): self.collect_roots() self.visit_all_objects() # + # Weakref support: clear the weak pointers to dying objects + if self.old_objects_with_weakrefs.non_empty(): + self.invalidate_old_weakrefs() + # # Finalizer support: adds the flag GCFLAG_VISITED to all objects # with a finalizer and all objects reachable from there (and also # moves some objects from 'objects_with_finalizers' to @@ -1149,10 +1218,6 @@ class MiniMarkGC(MovingGCBase): # self.objects_to_trace.delete() # - # Weakref support: clear the weak pointers to dying objects - if self.old_objects_with_weakrefs.non_empty(): - self.invalidate_old_weakrefs() - # # Walk all rawmalloced objects and free the ones that don't # have the GCFLAG_VISITED flag. self.free_unvisited_rawmalloc_objects() @@ -1182,8 +1247,8 @@ class MiniMarkGC(MovingGCBase): # have allocated 'major_collection_threshold' times more than # we currently have. bounded = self.set_major_threshold_from( - (self.get_total_memory_used() * self.major_collection_threshold) - + reserving_size) + self.get_total_memory_used() * self.major_collection_threshold, + reserving_size) # # Max heap size: gives an upper bound on the threshold. If we # already have at least this much allocated, raise MemoryError. @@ -1272,6 +1337,11 @@ class MiniMarkGC(MovingGCBase): self.run_finalizers.foreach(self._collect_obj, self.objects_to_trace) + def enumerate_all_roots(self, callback, arg): + self.prebuilt_root_objects.foreach(callback, arg) + MovingGCBase.enumerate_all_roots(self, callback, arg) + enumerate_all_roots._annspecialcase_ = 'specialize:arg(1)' + @staticmethod def _collect_obj(obj, objects_to_trace): objects_to_trace.append(obj) @@ -1323,7 +1393,7 @@ class MiniMarkGC(MovingGCBase): if self.is_valid_gc_object(obj): if self.is_in_nursery(obj): # - # The object not a tagged pointer, and is it still in the + # The object is not a tagged pointer, and it is still in the # nursery. Find or allocate a "shadow" object, which is # where the object will be moved by the next minor # collection diff --git a/pypy/rpython/memory/gc/semispace.py b/pypy/rpython/memory/gc/semispace.py index 5ebaf58460..bf40e596ca 100644 --- a/pypy/rpython/memory/gc/semispace.py +++ b/pypy/rpython/memory/gc/semispace.py @@ -230,6 +230,10 @@ class SemiSpaceGC(MovingGCBase): while self.max_space_size > size: self.max_space_size >>= 1 + @classmethod + def JIT_minimal_size_in_nursery(cls): + return cls.object_minimal_size + def collect(self, gen=0): self.debug_check_consistency() self.semispace_collect() @@ -262,10 +266,10 @@ class SemiSpaceGC(MovingGCBase): if self.run_finalizers.non_empty(): self.update_run_finalizers() scan = self.scan_copied(scan) - if self.objects_with_finalizers.non_empty(): - scan = self.deal_with_objects_with_finalizers(scan) if self.objects_with_weakrefs.non_empty(): self.invalidate_weakrefs() + if self.objects_with_finalizers.non_empty(): + scan = self.deal_with_objects_with_finalizers(scan) self.update_objects_with_id() self.finished_full_collect() self.debug_check_consistency() @@ -689,15 +693,10 @@ class SemiSpaceGC(MovingGCBase): self._ll_typeid_map[idx].size += llmemory.raw_malloc_usage(totsize) self.trace(adr, self.track_heap_parent, adr) - def _track_heap_root(self, root): - self.track_heap(root.address[0]) + @staticmethod + def _track_heap_root(obj, self): + self.track_heap(obj) - def heap_stats_walk_roots(self): - self.root_walker.walk_roots( - SemiSpaceGC._track_heap_root, - SemiSpaceGC._track_heap_root, - SemiSpaceGC._track_heap_root) - def heap_stats(self): self._tracked_dict = self.AddressDict() max_tid = self.root_walker.gcdata.max_type_id @@ -710,7 +709,7 @@ class SemiSpaceGC(MovingGCBase): while i < max_tid: self._tracked_dict.add(llmemory.cast_ptr_to_adr(ll_typeid_map[i])) i += 1 - self.heap_stats_walk_roots() + self.enumerate_all_roots(SemiSpaceGC._track_heap_root, self) self._ll_typeid_map = lltype.nullptr(ARRAY_TYPEID_MAP) self._tracked_dict.delete() return ll_typeid_map diff --git a/pypy/rpython/memory/gc/test/test_minimark.py b/pypy/rpython/memory/gc/test/test_minimark.py index a936108265..1bd67baebc 100644 --- a/pypy/rpython/memory/gc/test/test_minimark.py +++ b/pypy/rpython/memory/gc/test/test_minimark.py @@ -28,3 +28,42 @@ def test_card_marking_bytes_for_length(): assert gc.card_marking_bytes_for_length(P+P+1) == 3 assert gc.card_marking_bytes_for_length(P+P+P+P+P+P+P+P) == 8 assert gc.card_marking_bytes_for_length(P+P+P+P+P+P+P+P+1) == 9 + +def test_set_major_threshold(): + gc = MiniMarkGC(None, major_collection_threshold=2.0, + growth_rate_max=1.5) + gc.min_heap_size = 100.0 + gc.max_heap_size = 300.0 + gc.next_major_collection_threshold = 0.0 + # first, we don't grow past min_heap_size + for i in range(5): + gc.set_major_threshold_from(100.0) + assert gc.next_major_collection_threshold == 100.0 + # then we grow a lot + b = gc.set_major_threshold_from(100 * 2.0) + assert b is False + assert gc.next_major_collection_threshold == 150.0 + b = gc.set_major_threshold_from(150 * 2.0) + assert b is False + assert gc.next_major_collection_threshold == 225.0 + b = gc.set_major_threshold_from(225 * 2.0) + assert b is True + assert gc.next_major_collection_threshold == 300.0 # max reached + b = gc.set_major_threshold_from(300 * 2.0) + assert b is True + assert gc.next_major_collection_threshold == 300.0 + # then we shrink instantly + b = gc.set_major_threshold_from(100.0) + assert b is False + assert gc.next_major_collection_threshold == 100.0 + # then we grow a bit + b = gc.set_major_threshold_from(100 * 1.25) + assert b is False + assert gc.next_major_collection_threshold == 125.0 + b = gc.set_major_threshold_from(125 * 1.25) + assert b is False + assert gc.next_major_collection_threshold == 156.25 + # check that we cannot shrink below min_heap_size + b = gc.set_major_threshold_from(42.7) + assert b is False + assert gc.next_major_collection_threshold == 100.0 diff --git a/pypy/rpython/memory/gctransform/framework.py b/pypy/rpython/memory/gctransform/framework.py index 67e3ae9c1b..f51e868d72 100644 --- a/pypy/rpython/memory/gctransform/framework.py +++ b/pypy/rpython/memory/gctransform/framework.py @@ -172,6 +172,7 @@ class FrameworkGCTransformer(GCTransformer): gcdata.static_root_nongcend = a_random_address # patched in finish() gcdata.static_root_end = a_random_address # patched in finish() gcdata.max_type_id = 13 # patched in finish() + gcdata.typeids_z = a_random_address # patched in finish() self.gcdata = gcdata self.malloc_fnptr_cache = {} @@ -212,6 +213,9 @@ class FrameworkGCTransformer(GCTransformer): data_classdef.generalize_attr( 'max_type_id', annmodel.SomeInteger()) + data_classdef.generalize_attr( + 'typeids_z', + annmodel.SomeAddress()) annhelper = annlowlevel.MixLevelHelperAnnotator(self.translator.rtyper) @@ -415,6 +419,11 @@ class FrameworkGCTransformer(GCTransformer): [s_gc, annmodel.SomeInteger()], annmodel.s_Bool, minimal_transform=False) + self.get_typeids_z_ptr = getfn(inspector.get_typeids_z, + [s_gc], + annmodel.SomePtr( + lltype.Ptr(rgc.ARRAY_OF_CHAR)), + minimal_transform=False) self.set_max_heap_size_ptr = getfn(GCClass.set_max_heap_size.im_func, [s_gc, @@ -572,7 +581,14 @@ class FrameworkGCTransformer(GCTransformer): newgcdependencies = [] newgcdependencies.append(ll_static_roots_inside) ll_instance.inst_max_type_id = len(group.members) - self.write_typeid_list() + typeids_z = self.write_typeid_list() + ll_typeids_z = lltype.malloc(rgc.ARRAY_OF_CHAR, + len(typeids_z), + immortal=True) + for i in range(len(typeids_z)): + ll_typeids_z[i] = typeids_z[i] + ll_instance.inst_typeids_z = llmemory.cast_ptr_to_adr(ll_typeids_z) + newgcdependencies.append(ll_typeids_z) return newgcdependencies def get_finish_tables(self): @@ -599,6 +615,11 @@ class FrameworkGCTransformer(GCTransformer): for index in range(len(self.layoutbuilder.type_info_group.members)): f.write("member%-4d %s\n" % (index, all_ids.get(index, '?'))) f.close() + try: + import zlib + return zlib.compress(udir.join("typeids.txt").read(), 9) + except ImportError: + return '' def transform_graph(self, graph): func = getattr(graph, 'func', None) @@ -988,6 +1009,13 @@ class FrameworkGCTransformer(GCTransformer): resultvar=hop.spaceop.result) self.pop_roots(hop, livevars) + def gct_gc_typeids_z(self, hop): + livevars = self.push_roots(hop) + hop.genop("direct_call", + [self.get_typeids_z_ptr, self.c_const_gc], + resultvar=hop.spaceop.result) + self.pop_roots(hop, livevars) + def gct_malloc_nonmovable_varsize(self, hop): TYPE = hop.spaceop.result.concretetype if self.gcdata.gc.can_malloc_nonmovable(): @@ -1210,7 +1238,7 @@ def gen_zero_gc_pointers(TYPE, v, llops, previous_steps=None): sizeofaddr = llmemory.sizeof(llmemory.Address) -class BaseRootWalker: +class BaseRootWalker(object): need_root_stack = False def __init__(self, gctransformer): diff --git a/pypy/rpython/memory/support.py b/pypy/rpython/memory/support.py index e4386baa8d..4542c4a8a7 100644 --- a/pypy/rpython/memory/support.py +++ b/pypy/rpython/memory/support.py @@ -112,7 +112,7 @@ def get_address_stack(chunk_size=DEFAULT_CHUNK_SIZE, cache={}): cur = next free_non_gc_object(self) - def length(self): + def _length_estimate(self): chunk = self.chunk count = self.used_in_last_chunk while chunk: @@ -135,7 +135,7 @@ def get_address_stack(chunk_size=DEFAULT_CHUNK_SIZE, cache={}): foreach._annspecialcase_ = 'specialize:arg(1)' def stack2dict(self): - result = AddressDict(self.length()) + result = AddressDict(self._length_estimate()) self.foreach(_add_in_dict, result) return result diff --git a/pypy/rpython/memory/test/test_gc.py b/pypy/rpython/memory/test/test_gc.py index fe59bcefa3..4fe4d843be 100644 --- a/pypy/rpython/memory/test/test_gc.py +++ b/pypy/rpython/memory/test/test_gc.py @@ -278,6 +278,80 @@ class GCTest(object): res = self.interpret(f, []) assert res + def test_bug_1(self): + import weakref + class B(object): + pass + def g(): + b = B() + llop.gc__collect(lltype.Void) # force 'b' to be old + ref = weakref.ref(B()) + b.ref = ref + return ref + def f(): + ref = g() + llop.gc__collect(lltype.Void) + llop.gc__collect(lltype.Void) + result = (ref() is None) + return result + res = self.interpret(f, []) + assert res + + def test_cycle_with_weakref_and_del(self): + import weakref, gc + class A(object): + count = 0 + a = A() + class B(object): + def __del__(self): + # when __del__ is called, the weakref should have been cleared + if self.ref() is None: + a.count += 10 # ok + else: + a.count = 666 # not ok + class C(object): + pass + def g(): + c = C() + c.b = B() + ref = weakref.ref(c) + c.b.ref = ref + return ref + def f(): + ref = g() + llop.gc__collect(lltype.Void) + llop.gc__collect(lltype.Void) + result = a.count + (ref() is None) + return result + res = self.interpret(f, []) + assert res == 11 + + def test_weakref_to_object_with_finalizer_ordering(self): + import weakref, gc + class A(object): + count = 0 + a = A() + class B(object): + def __del__(self): + # when __del__ is called, the weakref should have been cleared + if self.ref() is None: + a.count += 10 # ok + else: + a.count = 666 # not ok + def g(): + b = B() + ref = weakref.ref(b) + b.ref = ref + return ref + def f(): + ref = g() + llop.gc__collect(lltype.Void) + llop.gc__collect(lltype.Void) + result = a.count + (ref() is None) + return result + res = self.interpret(f, []) + assert res == 11 + def test_id(self): class A(object): pass @@ -657,6 +731,9 @@ class TestMarkCompactGC(TestSemiSpaceGC): def test_finalizer_order(self): py.test.skip("Not implemented yet") + def test_weakref_to_object_with_finalizer_ordering(self): + py.test.skip("Not implemented yet") + class TestHybridGC(TestGenerationalGC): from pypy.rpython.memory.gc.hybrid import HybridGC as GCClass GC_CAN_MALLOC_NONMOVABLE = True diff --git a/pypy/rpython/rbuiltin.py b/pypy/rpython/rbuiltin.py index 86beda61c5..aa71174768 100644 --- a/pypy/rpython/rbuiltin.py +++ b/pypy/rpython/rbuiltin.py @@ -386,6 +386,14 @@ def rtype_free(hop, i_flavor, i_track_allocation=None): hop.exception_cannot_occur() hop.genop('free', vlist) +def rtype_render_immortal(hop, i_track_allocation=None): + vlist = [hop.inputarg(hop.args_r[0], arg=0)] + v_track_allocation = parse_kwds(hop, + (i_track_allocation, None)) + hop.exception_cannot_occur() + if i_track_allocation is None or v_track_allocation.value: + hop.genop('track_alloc_stop', vlist) + def rtype_const_result(hop): hop.exception_cannot_occur() return hop.inputconst(hop.r_result.lowleveltype, hop.s_result.const) @@ -523,6 +531,7 @@ def rtype_runtime_type_info(hop): BUILTIN_TYPER[lltype.malloc] = rtype_malloc BUILTIN_TYPER[lltype.free] = rtype_free +BUILTIN_TYPER[lltype.render_immortal] = rtype_render_immortal BUILTIN_TYPER[lltype.cast_primitive] = rtype_cast_primitive BUILTIN_TYPER[lltype.cast_pointer] = rtype_cast_pointer BUILTIN_TYPER[lltype.cast_opaque_ptr] = rtype_cast_opaque_ptr diff --git a/pypy/rpython/rmodel.py b/pypy/rpython/rmodel.py index 8c73bffcf4..49cb2f29e5 100644 --- a/pypy/rpython/rmodel.py +++ b/pypy/rpython/rmodel.py @@ -11,14 +11,14 @@ from pypy.rpython.error import TyperError, MissingRTypeOperation # initialization states for Repr instances -class setupstate: +class setupstate(object): NOTINITIALIZED = 0 INPROGRESS = 1 BROKEN = 2 FINISHED = 3 DELAYED = 4 -class Repr: +class Repr(object): """ An instance of Repr is associated with each instance of SomeXxx. It defines the chosen representation for the SomeXxx. The Repr subclasses generally follows the SomeXxx subclass hierarchy, but there are numerous diff --git a/pypy/rpython/test/test_rpbc.py b/pypy/rpython/test/test_rpbc.py index f919bbc6b5..e1621bcf46 100644 --- a/pypy/rpython/test/test_rpbc.py +++ b/pypy/rpython/test/test_rpbc.py @@ -1547,7 +1547,7 @@ class BaseTestRPBC(BaseRtypingTest): def test_always_raising_methods(self): class Base: def m(self): - raise NotImplementedError + raise KeyError class A(Base): def m(self): return 42 @@ -1560,11 +1560,11 @@ class BaseTestRPBC(BaseRtypingTest): o = B() try: o.m() - except NotImplementedError: - pass + except KeyError: + assert 0 return B().m() - self.interpret_raises(NotImplementedError, f, [7]) + self.interpret_raises(KeyError, f, [7]) def test_possible_missing_attribute_access(self): py.test.skip("Should explode or give some warning") diff --git a/pypy/tool/gcdump.py b/pypy/tool/gcdump.py index 92f1de3128..17ec585790 100755 --- a/pypy/tool/gcdump.py +++ b/pypy/tool/gcdump.py @@ -1,75 +1,96 @@ -#!/usr/bin/env python -""" Usage: gcdump.py gcdump typeids [outfile] +#! /usr/bin/env python """ +Prints a human-readable total out of a dumpfile produced +by gc.dump_rpy_heap(), and optionally a typeids.txt. -from __future__ import division -import re -import sys +Syntax: dump.py <dumpfile> [<typeids.txt>] -class GcDump(object): - def __init__(self, count, size, links): - self.count = count - self.size = size - self.links = links +By default, typeids.txt is loaded from the same dir as dumpfile. +""" +import sys, array, struct, os + + +class Stat(object): + summary = {} + typeids = {0: '<GCROOT>'} + + def summarize(self, filename): + a = self.load_dump_file(filename) + self.summary = {} # {typenum: [count, totalsize]} + for obj in self.walk(a): + self.add_object_summary(obj[2], obj[3]) + + def load_typeids(self, filename): + self.typeids = Stat.typeids.copy() + for num, line in enumerate(open(filename)): + if num == 0: + continue + words = line.split() + if words[0].startswith('member'): + del words[0] + if words[0] == 'GcStruct': + del words[0] + self.typeids[num] = ' '.join(words) + + def get_type_name(self, num): + return self.typeids.get(num, '<typenum %d>' % num) -def read_gcdump(f): - lines = f.readlines() - r = [None] * len(lines) - for i, line in enumerate(lines): - count, size, rest = line.split(" ") - r[i] = GcDump(int(count), int(size), - [int(j) for j in rest.split(",")]) - return r + def print_summary(self): + items = self.summary.items() + items.sort(key=lambda(typenum, stat): stat[1]) # sort by totalsize + totalsize = 0 + for typenum, stat in items: + totalsize += stat[1] + print '%8d %8.2fM %s' % (stat[0], stat[1] / (1024.0*1024.0), + self.get_type_name(typenum)) + print 'total %.1fM' % (totalsize / (1024.0*1024.0),) -def read_typeids(f): - res = [] - for line in f.readlines(): - member, name = re.split("\s+", line, 1) - assert member == "member%d" % len(res) - res.append(name.strip("\n")) - return res + def load_dump_file(self, filename): + f = open(filename, 'rb') + f.seek(0, 2) + end = f.tell() + f.seek(0) + a = array.array('l') + a.fromfile(f, end / struct.calcsize('l')) + f.close() + return a -def getname(name, _cache = {}): - try: - return _cache[name] - except KeyError: - no = len(_cache) - _cache[name] = '(%d)' % len(_cache) - return '(%d) %s' % (no, name) + def add_object_summary(self, typenum, sizeobj): + try: + stat = self.summary[typenum] + except KeyError: + stat = self.summary[typenum] = [0, 0] + stat[0] += 1 + stat[1] += sizeobj -def process(f, gcdump, typeids): - f.write("events: number B\n\n") - for tid, name in enumerate(typeids): - if not tid % 100: - sys.stderr.write("%d%%.." % (tid / len(typeids) * 100)) - f.write("fn=%s\n" % getname(name)) - f.write("0 %d %d\n" % (gcdump[tid].count, gcdump[tid].size)) - for subtid, no in enumerate(gcdump[tid].links): - if no != 0: - f.write("cfn=%s\n" % getname(typeids[subtid])) - f.write("calls=0 %d\n" % no) - f.write("0 %d %d\n" % (gcdump[subtid].count, - gcdump[subtid].size)) - f.write("\n") - sys.stderr.write("100%\n") + def walk(self, a, start=0, stop=None): + assert a[-1] == -1, "invalid or truncated dump file (or 32/64-bit mix)" + assert a[-2] != -1, "invalid or truncated dump file (or 32/64-bit mix)" + print >> sys.stderr, 'walking...', + i = start + if stop is None: + stop = len(a) + while i < stop: + j = i + 3 + while a[j] != -1: + j += 1 + yield (i, a[i], a[i+1], a[i+2], a[i+3:j]) + i = j + 1 + print >> sys.stderr, 'done' -def main(gcdump_f, typeids_f, outfile): - gcdump = read_gcdump(gcdump_f) - gcdump_f.close() - typeids = read_typeids(typeids_f) - typeids_f.close() - process(outfile, gcdump, typeids) if __name__ == '__main__': - if len(sys.argv) == 4: - outfile = open(sys.argv[3], "w") - elif len(sys.argv) == 3: - outfile = sys.stdout + if len(sys.argv) <= 1: + print >> sys.stderr, __doc__ + sys.exit(2) + stat = Stat() + stat.summarize(sys.argv[1]) + # + if len(sys.argv) > 2: + typeid_name = sys.argv[2] else: - print __doc__ - sys.exit(1) - gcdump = open(sys.argv[1]) - typeids = open(sys.argv[2]) - main(gcdump, typeids, outfile) - if len(sys.argv) == 4: - outfile.close() + typeid_name = os.path.join(os.path.dirname(sys.argv[1]), 'typeids.txt') + if os.path.isfile(typeid_name): + stat.load_typeids(typeid_name) + # + stat.print_summary() diff --git a/pypy/translator/c/gcc/trackgcroot.py b/pypy/translator/c/gcc/trackgcroot.py index e1d82632cc..3a33f4a61c 100755 --- a/pypy/translator/c/gcc/trackgcroot.py +++ b/pypy/translator/c/gcc/trackgcroot.py @@ -1905,7 +1905,7 @@ def getidentifier(s): if __name__ == '__main__': - verbose = 1 + verbose = 0 shuffle = False output_raw_table = False if sys.platform == 'darwin': diff --git a/pypy/translator/c/genc.py b/pypy/translator/c/genc.py index c507ec633c..f4948f430b 100644 --- a/pypy/translator/c/genc.py +++ b/pypy/translator/c/genc.py @@ -553,7 +553,7 @@ class CStandaloneBuilder(CBuilder): ('debug', '', '$(MAKE) CFLAGS="$(DEBUGFLAGS) -DRPY_ASSERT" $(TARGET)'), ('debug_exc', '', '$(MAKE) CFLAGS="$(DEBUGFLAGS) -DRPY_ASSERT -DDO_LOG_EXC" $(TARGET)'), ('debug_mem', '', '$(MAKE) CFLAGS="$(DEBUGFLAGS) -DRPY_ASSERT -DTRIVIAL_MALLOC_DEBUG" $(TARGET)'), - ('no_obmalloc', '', '$(MAKE) CFLAGS="-g -O1 -DRPY_ASSERT -DNO_OBMALLOC" $(TARGET)'), + ('no_obmalloc', '', '$(MAKE) CFLAGS="-g -O2 -DRPY_ASSERT -DNO_OBMALLOC" $(TARGET)'), ('linuxmemchk', '', '$(MAKE) CFLAGS="$(DEBUGFLAGS) -DRPY_ASSERT -DLINUXMEMCHK" $(TARGET)'), ('llsafer', '', '$(MAKE) CFLAGS="-O2 -DRPY_LL_ASSERT" $(TARGET)'), ('lldebug', '', '$(MAKE) CFLAGS="$(DEBUGFLAGS) -DRPY_ASSERT -DRPY_LL_ASSERT" $(TARGET)'), @@ -623,9 +623,15 @@ class CStandaloneBuilder(CBuilder): mk.definition('OBJECTS', '$(ASMLBLFILES) gcmaptable.s') mk.rule('%.s', '%.c', '$(CC) $(CFLAGS) $(CFLAGSEXTRA) -frandom-seed=$< -o $@ -S $< $(INCLUDEDIRS)') mk.rule('%.lbl.s %.gcmap', '%.s', - python + '$(PYPYDIR)/translator/c/gcc/trackgcroot.py -m$(PYPY_MAIN_FUNCTION) -t $< > $*.gcmap') + [python + + '$(PYPYDIR)/translator/c/gcc/trackgcroot.py ' + '-m$(PYPY_MAIN_FUNCTION) -t $< > $*.gctmp', + 'mv $*.gctmp $*.gcmap']) mk.rule('gcmaptable.s', '$(GCMAPFILES)', - python + '$(PYPYDIR)/translator/c/gcc/trackgcroot.py $(GCMAPFILES) > $@') + [python + + '$(PYPYDIR)/translator/c/gcc/trackgcroot.py ' + '$(GCMAPFILES) > $@.tmp', + 'mv $@.tmp $@']) mk.rule('.PRECIOUS', '%.s', "# don't remove .s files if Ctrl-C'ed") else: diff --git a/pypy/translator/c/node.py b/pypy/translator/c/node.py index c9c68b02ff..3fa45b06f4 100644 --- a/pypy/translator/c/node.py +++ b/pypy/translator/c/node.py @@ -714,7 +714,11 @@ class ArrayNode(ContainerNode): s = ''.join([self.obj.getitem(i) for i in range(len(self.obj.items))]) else: s = ''.join(self.obj.items) - yield '\t%s%s' % (length, c_char_array_constant(s)) + array_constant = c_char_array_constant(s) + if array_constant.startswith('{') and barebonearray(T): + assert array_constant.endswith('}') + array_constant = array_constant[1:-1].strip() + yield '\t%s%s' % (length, array_constant) yield '}' else: barebone = barebonearray(T) diff --git a/pypy/translator/c/src/asm_gcc_x86.h b/pypy/translator/c/src/asm_gcc_x86.h index ab6a7f4301..c8be25407b 100644 --- a/pypy/translator/c/src/asm_gcc_x86.h +++ b/pypy/translator/c/src/asm_gcc_x86.h @@ -2,6 +2,8 @@ * It replaces some complex macros with native assembler instructions. */ +#if 0 /* --- disabled: does not give any speed-up --- */ + #undef OP_INT_ADD_OVF #define OP_INT_ADD_OVF(x,y,r) \ asm volatile( \ @@ -50,6 +52,13 @@ : "0"(x), "g"(y) /* inputs */ \ : "cc", "memory") /* clobber */ +extern void op_int_overflowed(void) + asm ("_op_int_overflowed") + __attribute__((used)); + +#endif /* 0 */ + + /* Pentium only! */ #define READ_TIMESTAMP(val) \ asm volatile("rdtsc" : "=A" (val)) @@ -62,19 +71,15 @@ // I don't know how important it is, comment talks about time warps -/* prototypes */ - -extern void op_int_overflowed(void) - asm ("_op_int_overflowed") - __attribute__((used)); - /* implementations */ #ifndef PYPY_NOT_MAIN_FILE +# if 0 /* disabled */ void op_int_overflowed(void) { FAIL_OVF("integer operation"); } +# endif #endif diff --git a/pypy/translator/c/src/g_include.h b/pypy/translator/c/src/g_include.h index 490ac718a5..026252a565 100644 --- a/pypy/translator/c/src/g_include.h +++ b/pypy/translator/c/src/g_include.h @@ -39,10 +39,9 @@ #include "src/instrument.h" /* optional assembler bits */ -// disabled: does not give any speed-up -//#if defined(__GNUC__) && defined(__i386__) -//# include "src/asm_gcc_x86.h" -//#endif +#if defined(__GNUC__) && defined(__i386__) +# include "src/asm_gcc_x86.h" +#endif #if defined(__GNUC__) && defined(__ppc__) # include "src/asm_ppc.h" diff --git a/pypy/translator/c/src/signals.h b/pypy/translator/c/src/signals.h index 041bc94834..f6a2e1c2a3 100644 --- a/pypy/translator/c/src/signals.h +++ b/pypy/translator/c/src/signals.h @@ -47,16 +47,12 @@ void pypysig_setflag(int signum); /* signal will set a flag which can be /* utility to poll for signals that arrived */ int pypysig_poll(void); /* => signum or -1 */ -/* When a signal is received, the bit 30 of pypysig_occurred is set. - After all signals are processed by pypysig_poll(), the bit 30 is - cleared again. The variable is exposed and RPython code is free to - use the other bits in any way. */ -#define PENDING_SIGNAL_BIT (1 << 30) +/* When a signal is received, pypysig_counter is set to -1. */ /* This is a struct for the JIT. See interp_signal.py. */ struct pypysig_long_struct { long value; }; -extern struct pypysig_long_struct pypysig_occurred; +extern struct pypysig_long_struct pypysig_counter; /* some C tricks to get/set the variable as efficiently as possible: use macros when compiling as a stand-alone program, but still @@ -64,18 +60,20 @@ extern struct pypysig_long_struct pypysig_occurred; #undef pypysig_getaddr_occurred void *pypysig_getaddr_occurred(void); #ifndef PYPY_NOT_MAIN_FILE -void *pypysig_getaddr_occurred(void) { return (void *)(&pypysig_occurred); } +void *pypysig_getaddr_occurred(void) { return (void *)(&pypysig_counter); } #endif -#define pypysig_getaddr_occurred() ((void *)(&pypysig_occurred)) +#define pypysig_getaddr_occurred() ((void *)(&pypysig_counter)) /************************************************************/ /* Implementation */ #ifndef PYPY_NOT_MAIN_FILE -struct pypysig_long_struct pypysig_occurred; -static volatile long *pypysig_occurred_v = (volatile long *)&pypysig_occurred.value; -static volatile int pypysig_flags[NSIG]; +struct pypysig_long_struct pypysig_counter = {0}; +static char volatile pypysig_flags[NSIG] = {0}; +static int volatile pypysig_occurred = 0; +/* pypysig_occurred is only an optimization: it tells if any + pypysig_flags could be set. */ void pypysig_ignore(int signum) { @@ -108,10 +106,11 @@ void pypysig_default(int signum) static void signal_setflag_handler(int signum) { if (0 <= signum && signum < NSIG) + { pypysig_flags[signum] = 1; - /* the point of "*pypysig_occurred_v" instead of just "pypysig_occurred" - is the volatile declaration */ - *pypysig_occurred_v |= PENDING_SIGNAL_BIT; + pypysig_occurred = 1; + pypysig_counter.value = -1; + } } void pypysig_setflag(int signum) @@ -130,27 +129,21 @@ void pypysig_setflag(int signum) int pypysig_poll(void) { - /* the two commented out lines below are useful for performance in - normal usage of pypysig_poll(); however, pypy/module/signal/ is - not normal usage. It only calls pypysig_poll() if the - PENDING_SIGNAL_BIT is set, and it clears that bit first. */ - -/* if (pypysig_occurred & PENDING_SIGNAL_BIT) */ + if (pypysig_occurred) { - int i; -/* pypysig_occurred &= ~PENDING_SIGNAL_BIT; */ - for (i=0; i<NSIG; i++) - if (pypysig_flags[i]) - { - pypysig_flags[i] = 0; - /* maybe another signal is pending: */ - pypysig_occurred.value |= PENDING_SIGNAL_BIT; - return i; - } + int i; + pypysig_occurred = 0; + for (i=0; i<NSIG; i++) + if (pypysig_flags[i]) + { + pypysig_flags[i] = 0; + pypysig_occurred = 1; /* maybe another signal is pending */ + return i; + } } - return -1; /* no pending signal */ + return -1; /* no pending signal */ } -#endif +#endif /* !PYPY_NOT_MAIN_FILE */ #endif diff --git a/pypy/translator/c/test/test_lltyped.py b/pypy/translator/c/test/test_lltyped.py index 1f8793ecdc..a2576ed874 100644 --- a/pypy/translator/c/test/test_lltyped.py +++ b/pypy/translator/c/test/test_lltyped.py @@ -1,6 +1,7 @@ import py from pypy.rpython.lltypesystem.lltype import * from pypy.translator.c.test import test_typed +from pypy.tool.sourcetools import func_with_new_name class TestLowLevelType(test_typed.CompilationTestCase): @@ -655,6 +656,45 @@ class TestLowLevelType(test_typed.CompilationTestCase): fn = self.getcompiled(llf) fn() + def test_prebuilt_raw_arrays(self): + from pypy.rpython.lltypesystem import rffi, ll2ctypes + # + def make_test_function(cast, haslength, length): + a = malloc(A, length, flavor='raw', immortal=True) + # two cases: a zero-terminated array if length == 6 or 1030, + # a non-zero-terminated array if length == 557 or 1031 + for i in range(length): + a[i] = cast(256 - 5 + i) + def llf(): + for i in range(length): + if a[i] != cast(256 - 5 + i): + return False + if haslength and len(a) != length: + return False + return True + return func_with_new_name(llf, repr((A, haslength, length))) + # + testfns = [] + records = [] + for OF, cast in [(Void, lambda n: None), + (Char, lambda n: chr(n & 0xFF)), + (Signed, lambda n: n)]: + for A, haslength in [(rffi.CArray(OF), False), + (Array(OF), True)]: + for length in [0, 6, 557, 1030, 1031]: + testfns.append(make_test_function(cast, haslength, length)) + records.append((A, haslength, length)) + def llf(): + i = 0 + for fn in testfns: + if not fn(): + return i # returns the index of the failing function + i += 1 + return -42 + fn = self.getcompiled(llf) + res = fn() + assert res == -42, "failing function: %r" % (records[res],) + def test_prebuilt_ll2ctypes_array(self): from pypy.rpython.lltypesystem import rffi, ll2ctypes A = rffi.CArray(Char) @@ -841,3 +881,17 @@ class TestLowLevelType(test_typed.CompilationTestCase): assert res == -98765432 res = fn(1) assert res == -9999999 + + def test_render_immortal(self): + A = FixedSizeArray(Signed, 1) + a1 = malloc(A, flavor='raw') + render_immortal(a1) + a1[0] = 42 + def llf(): + a2 = malloc(A, flavor='raw') + render_immortal(a2) + a2[0] = 3 + return a1[0] + a2[0] + fn = self.getcompiled(llf) + assert fn() == 45 + diff --git a/pypy/translator/c/test/test_newgc.py b/pypy/translator/c/test/test_newgc.py index 053f5d01db..2527aaed39 100644 --- a/pypy/translator/c/test/test_newgc.py +++ b/pypy/translator/c/test/test_newgc.py @@ -1066,7 +1066,9 @@ class TestUsingFramework(object): filename_dump = str(udir.join('test_dump_rpy_heap')) def define_dump_rpy_heap(self): - U = lltype.GcStruct('U', ('x', lltype.Signed)) + U = lltype.GcForwardReference() + U.become(lltype.GcStruct('U', ('next', lltype.Ptr(U)), + ('x', lltype.Signed))) S = lltype.GcStruct('S', ('u', lltype.Ptr(U))) A = lltype.GcArray(lltype.Ptr(S)) filename = self.filename_dump @@ -1074,11 +1076,16 @@ class TestUsingFramework(object): def fn(): s = lltype.malloc(S) s.u = lltype.malloc(U) + s.u.next = lltype.malloc(U) + s.u.next.next = lltype.malloc(U) a = lltype.malloc(A, 1000) s2 = lltype.malloc(S) # fd = os.open(filename, os.O_WRONLY | os.O_CREAT, 0666) rgc.dump_rpy_heap(fd) + keepalive_until_here(s2) + keepalive_until_here(s) + keepalive_until_here(a) os.close(fd) return 0 @@ -1087,8 +1094,43 @@ class TestUsingFramework(object): def test_dump_rpy_heap(self): self.run("dump_rpy_heap") assert os.path.exists(self.filename_dump) - assert os.path.getsize(self.filename_dump) > 0 # minimal test + assert os.path.getsize(self.filename_dump) > 64 + + filename_dump_typeids_z = str(udir.join('test_typeids_z')) + def define_write_typeids_z(self): + U = lltype.GcForwardReference() + U.become(lltype.GcStruct('U', ('next', lltype.Ptr(U)), + ('x', lltype.Signed))) + S = lltype.GcStruct('S', ('u', lltype.Ptr(U))) + A = lltype.GcArray(lltype.Ptr(S)) + filename = self.filename_dump_typeids_z + + def fn(): + s = lltype.malloc(S) + s.u = lltype.malloc(U) + s.u.next = lltype.malloc(U) + s.u.next.next = lltype.malloc(U) + a = lltype.malloc(A, 1000) + s2 = lltype.malloc(S) + # + p = rgc.get_typeids_z() + s = ''.join([p[i] for i in range(len(p))]) + fd = os.open(filename, os.O_WRONLY | os.O_CREAT, 0666) + os.write(fd, s) + os.close(fd) + return 0 + + return fn + def test_write_typeids_z(self): + self.run("write_typeids_z") + f = open(self.filename_dump_typeids_z) + data_z = f.read() + f.close() + import zlib + data = zlib.decompress(data_z) + assert data.startswith('member0') + assert 'GcArray of * GcStruct S {' in data class TestSemiSpaceGC(TestUsingFramework, snippet.SemiSpaceGCTestDefines): gcpolicy = "semispace" @@ -1173,21 +1215,22 @@ class TestSemiSpaceGC(TestUsingFramework, snippet.SemiSpaceGCTestDefines): b = 0 c = 0 for i in range(len(tb)): - if tb[i].count == 10: + if tb[i].count == 10: # the type of S a += 1 nr = i for i in range(len(tb)): - if tb[i].count == 3: + if tb[i].count == 3: # the type GcArray(Ptr(S)) b += 1 c += tb[i].links[nr] - # we don't count b here since there can be more singletons, + # b can be 1 or 2 here since _heap_stats() is free to return or + # ignore the three GcStructs that point to the GcArray(Ptr(S)). # important one is c, a is for check return c * 100 + b * 10 + a return f def test_gc_heap_stats(self): res = self.run("gc_heap_stats") - assert res == 3011 + assert res == 3011 or res == 3021 def definestr_string_builder(cls): def fn(_): diff --git a/pypy/translator/cli/src/pypylib.cs b/pypy/translator/cli/src/pypylib.cs index 000ca47f35..782334f6e3 100644 --- a/pypy/translator/cli/src/pypylib.cs +++ b/pypy/translator/cli/src/pypylib.cs @@ -83,8 +83,12 @@ namespace pypy.test return Double.NegativeInfinity; else if (s == "nan") return Double.NaN; - else - return System.Convert.ToDouble(s); + else { + System.Globalization.NumberFormatInfo formatter; + formatter = new System.Globalization.NumberFormatInfo(); + formatter.NumberDecimalSeparator = "."; + return System.Convert.ToDouble(s, formatter); + } } } diff --git a/pypy/translator/goal/app_main.py b/pypy/translator/goal/app_main.py index b9dcb7f251..4a1a5b7083 100644 --- a/pypy/translator/goal/app_main.py +++ b/pypy/translator/goal/app_main.py @@ -326,10 +326,6 @@ def run_command_line(go_interactive, except: print >> sys.stderr, "'import site' failed" - # update sys.path *after* loading site.py, in case there is a - # "site.py" file in the script's directory. - sys.path.insert(0, '') - if warnoptions: sys.warnoptions.append(warnoptions) from warnings import _processoptions @@ -366,6 +362,9 @@ def run_command_line(go_interactive, try: if run_command: # handle the "-c" command + # Put '' on sys.path + sys.path.insert(0, '') + def run_it(): exec cmd in mainmodule.__dict__ success = run_toplevel(run_it) @@ -378,6 +377,13 @@ def run_command_line(go_interactive, elif run_stdin: # handle the case where no command/filename/module is specified # on the command-line. + + # update sys.path *after* loading site.py, in case there is a + # "site.py" file in the script's directory. Only run this if we're + # executing the interactive prompt, if we're running a script we + # put it's directory on sys.path + sys.path.insert(0, '') + if go_interactive or sys.stdin.isatty(): # If stdin is a tty or if "-i" is specified, we print # a banner and run $PYTHONSTARTUP. diff --git a/pypy/translator/goal/targetpypystandalone.py b/pypy/translator/goal/targetpypystandalone.py index 5b1389f948..1a2d01109e 100644 --- a/pypy/translator/goal/targetpypystandalone.py +++ b/pypy/translator/goal/targetpypystandalone.py @@ -150,6 +150,9 @@ class PyPyTarget(object): if config.objspace.allworkingmodules: from pypy.config.pypyoption import enable_allworkingmodules enable_allworkingmodules(config) + if config.objspace.translationmodules: + from pypy.config.pypyoption import enable_translationmodules + enable_translationmodules(config) if config.translation.type_system == 'ootype': config.objspace.usemodules.suggest(rbench=True) diff --git a/pypy/translator/goal/test2/test_app_main.py b/pypy/translator/goal/test2/test_app_main.py index 1fc6a7f489..5c437b22d6 100644 --- a/pypy/translator/goal/test2/test_app_main.py +++ b/pypy/translator/goal/test2/test_app_main.py @@ -95,6 +95,11 @@ class TestInteraction: child.expect('>>> ') child.sendline('__name__') child.expect("'__main__'") + child.expect('>>> ') + child.sendline('import sys') + child.expect('>>> ') + child.sendline("'' in sys.path") + child.expect("True") def test_run_script(self): child = self.spawn([demo_script]) @@ -463,8 +468,10 @@ class TestNonInteractive: yield finally: old_cwd.chdir() - os.putenv('PYTHONPATH', old_pythonpath) - + # Can't call putenv with a None argument. + if old_pythonpath is not None: + os.putenv('PYTHONPATH', old_pythonpath) + tmpdir.join('site.py').write('print "SHOULD NOT RUN"') runme_py = tmpdir.join('runme.py') runme_py.write('print "some text"') @@ -485,9 +492,12 @@ class TestNonInteractive: with chdir_and_unset_pythonpath(tmpdir): data = self.run(cmdline2, python_flags='-S') - assert data.startswith("some new text\n") assert repr(str(tmpdir.join('otherpath'))) in data + assert "''" not in data + + data = self.run('-c "import sys; print sys.path"') + assert data.startswith("[''") class AppTestAppMain: |