TechLead
Lesson 10 of 25
5 min read
Python

Decorators and Generators

Master Python decorators for metaprogramming and generators for memory-efficient iteration

Understanding Decorators

A decorator is a function that takes another function as input and returns a modified version of it. Decorators are Python's implementation of the decorator design pattern and are used extensively in frameworks like Flask, Django, and FastAPI. They use the @ syntax for clean application.

import functools
import time

# Basic decorator
def timer(func):
    @functools.wraps(func)  # Preserves function metadata
    def wrapper(*args, **kwargs):
        start = time.perf_counter()
        result = func(*args, **kwargs)
        elapsed = time.perf_counter() - start
        print(f"{func.__name__} took {elapsed:.4f}s")
        return result
    return wrapper

@timer
def slow_function():
    time.sleep(1)
    return "done"

result = slow_function()  # "slow_function took 1.0012s"

# Decorator with arguments
def retry(max_attempts=3, delay=1):
    def decorator(func):
        @functools.wraps(func)
        def wrapper(*args, **kwargs):
            for attempt in range(1, max_attempts + 1):
                try:
                    return func(*args, **kwargs)
                except Exception as e:
                    if attempt == max_attempts:
                        raise
                    print(f"Attempt {attempt} failed: {e}. Retrying...")
                    time.sleep(delay)
        return wrapper
    return decorator

@retry(max_attempts=3, delay=0.5)
def unreliable_api_call():
    import random
    if random.random() < 0.7:
        raise ConnectionError("API unavailable")
    return {"status": "success"}

Practical Decorator Patterns

import functools

# Caching decorator (memoization)
def memoize(func):
    cache = {}
    @functools.wraps(func)
    def wrapper(*args):
        if args not in cache:
            cache[args] = func(*args)
        return cache[args]
    return wrapper

@memoize
def fibonacci(n):
    if n < 2:
        return n
    return fibonacci(n - 1) + fibonacci(n - 2)

print(fibonacci(100))  # Instant! Without memoize this would take forever

# Python has a built-in version: functools.lru_cache
@functools.lru_cache(maxsize=128)
def expensive_computation(n):
    return sum(i ** 2 for i in range(n))

# Logging decorator
def log_calls(func):
    @functools.wraps(func)
    def wrapper(*args, **kwargs):
        args_repr = [repr(a) for a in args]
        kwargs_repr = [f"{k}={v!r}" for k, v in kwargs.items()]
        signature = ", ".join(args_repr + kwargs_repr)
        print(f"Calling {func.__name__}({signature})")
        result = func(*args, **kwargs)
        print(f"{func.__name__} returned {result!r}")
        return result
    return wrapper

@log_calls
def add(a, b):
    return a + b

add(3, 4)
# Calling add(3, 4)
# add returned 7

# Stacking decorators
@timer
@log_calls
def process_data(data):
    return sorted(data)

Generators

Generators are functions that produce a sequence of values lazily using yield. Instead of creating an entire list in memory, they generate values one at a time. This makes them extremely memory-efficient for large datasets or infinite sequences.

# Basic generator
def count_up(start, end):
    current = start
    while current <= end:
        yield current
        current += 1

for num in count_up(1, 5):
    print(num)  # 1, 2, 3, 4, 5

# Generator for large data processing
def read_large_file(file_path):
    """Read a file line by line without loading it all into memory."""
    with open(file_path, "r") as f:
        for line in f:
            yield line.strip()

# Infinite generator
def fibonacci_gen():
    a, b = 0, 1
    while True:
        yield a
        a, b = b, a + b

# Take first 10 Fibonacci numbers
from itertools import islice
fib = fibonacci_gen()
first_10 = list(islice(fib, 10))
print(first_10)  # [0, 1, 1, 2, 3, 5, 8, 13, 21, 34]

# Generator expressions (like list comprehensions but lazy)
squares_list = [x**2 for x in range(1000000)]   # Creates entire list in memory
squares_gen = (x**2 for x in range(1000000))     # Generates values on demand

# Memory comparison
import sys
print(sys.getsizeof(squares_list))  # ~8 MB
print(sys.getsizeof(squares_gen))   # ~200 bytes

# Practical: data pipeline with generators
def parse_logs(lines):
    for line in lines:
        parts = line.split(" | ")
        yield {"timestamp": parts[0], "level": parts[1], "message": parts[2]}

def filter_errors(records):
    for record in records:
        if record["level"] == "ERROR":
            yield record

# Chain generators for memory-efficient pipeline
# errors = filter_errors(parse_logs(read_large_file("app.log")))
# for error in errors:
#     print(error["message"])

yield from and Generator Delegation

# yield from delegates to a sub-generator
def flatten(nested_list):
    for item in nested_list:
        if isinstance(item, list):
            yield from flatten(item)  # Recursive delegation
        else:
            yield item

nested = [1, [2, 3], [4, [5, 6]], 7]
print(list(flatten(nested)))  # [1, 2, 3, 4, 5, 6, 7]

# Combining multiple generators
def chain_generators(*generators):
    for gen in generators:
        yield from gen

gen1 = (x for x in range(3))
gen2 = (x for x in range(10, 13))
combined = list(chain_generators(gen1, gen2))
print(combined)  # [0, 1, 2, 10, 11, 12]

Key Takeaways

  • Use functools.wraps: Always preserve the wrapped function's metadata
  • Generators save memory: Use yield for large or infinite sequences
  • Generator expressions: Use (x for x in ...) instead of [x for x in ...] for large data
  • Pipeline pattern: Chain generators for memory-efficient data processing

Continue Learning