Using Decorators with MapReduce Mapper / Reducer Functions?

Question

I am trying to wrap my mapper / reducer functions with something like:

def log_exceptions_to_sentry(sentry_id, raise_exception):
    def decorator(fn):
        def wrapper(*args, **kwargs):
            try:
                return fn(*args, **kwargs)
            except Exception, e:
                client = Client(sentry_id)
                client.captureException(
                    exc_info=sys.exc_info())
                if raise_exception:
                    raise e
        return wrapper
    return decorator

and so my translator / gear functions are as follows:

@log_exceptions_to_sentry(SENTRY_ID, False)
def my_mapper_fn(item):
    logging.info(item)

But that does not work. Without a decorator, I would find INFOlogs item. But if I put a decorator, it seems that the functions of the translator / gearbox are not called at all.

I was hoping to make it easier to record any errors that my functions might encounter, so I can fix them, as it is almost impossible to find MapReduce through the AppEngine logs.

I could wrap the body of the entire function in a block try ... except, but the decorator would be cleaner.

Answer 1

, . partuclar, ,

try:
    return fn(*args, **kwargs)

try:
    fn(*args, **kwargs)

, , : http://simeonfranklin.com/blog/2012/jul/1/python-decorators-in-12-steps/

- , , , :

sentry_id = id
raise_exception = 1

def basic_decorator(function):
    global sentry_id, raise_exception
    def wrapper(*args,**kwargs):
        try:
            function(*args,**kwargs)
        except Exception, e:
            client = Client(sentry_id)
            client.captureException(exc_info=sys.exc_info())
            if raise_exception:
                raise
    return wrapper

@basic_decorator
def my_mapper_fn(item):
    logging.info(item)

sentry_id raise_exception, . , sentry_id, raise_exception function . :

def log_exceptions_to_sentry(sentry_id,raise_exception=1):
    def basic_decorator(function):
        def wrapper(*args, **kwargs):
            try:
                function(*args,**kwargs)
            except Exception, e:
                client = Client(sentry_id)
                client.captureException(exc_info=sys.exc_info())
                if raise_exception:
                    raise

        return wrapper
    return basic_decorator

@log_exceptions_to_sentry(SENTRY_ID,RAISE_EXCEPTION)
def my_mapper_fn(item):
    logging.info(item)

Answer 2

, SENTRY_ID Client, . . , , . , , .

SENTRY_ID = 1
class Client(object):
    def __init__(self, sentry_id): pass
    def captureException(self, **kwargs):
        print('captureException, ', kwargs['exc_info'])

def log_exceptions_to_sentry(sentry_id, raise_exception):
    def decorator(fn):
        def wrapper(*args, **kwargs):
            try:
                return fn(*args, **kwargs)
            except Exception as e:
                client = Client(sentry_id)
                client.captureException(
                    exc_info=sys.exc_info())
                if raise_exception:
                    raise e
        return wrapper
    return decorator

def fn(item):
    logging.debug(item)
    logging.info(item)
    logging.error(item)

@log_exceptions_to_sentry(SENTRY_ID, False)
def my_mapper_fn(item):
    logging.debug(item)
    logging.info(item)
    logging.error(item)
    return 1

@log_exceptions_to_sentry(SENTRY_ID, False)
def my_mapper_fn2(item):
    raise Exception()

logging.basicConfig(
    level = logging.INFO,
    format = '%(levelname)s:%(name)s:%(message)s',
    #format = '%(message)s',
)

x = fn({'a':1})
print(x)
x = my_mapper_fn({'b':2})
print(x)
x = my_mapper_fn2({'c':3})
print(x)

:

INFO:root:{'a': 1}
ERROR:root:{'a': 1}
None
INFO:root:{'b': 2}
ERROR:root:{'b': 2}
1
captureException,  (<type 'exceptions.Exception'>, Exception(), <traceback object at 0x1813cf8>)
None