Upgrade to Pro — share decks privately, control downloads, hide ads and more …

Redis Hacks

Redis Hacks

Python Nordeste 2014 - Lightning Talk

51567a4f786cd8a2c41c513b592de9f9?s=128

David Cramer

May 03, 2014
Tweet

Transcript

  1. David Cramer twitter.com/zeeg Redis Hacks (or “How Sentry Scales”)

  2. Buffering Writes

  3. r = Redis() ! def incr(type, id): key = 'pending:{}'.format(type)

    ! r.zincrby(key, id, 1)
  4. r = Redis() ! def flush(type): key = 'pending:{}'.format(type) result

    = r.zrange(key, 0, -1, withscores=True) ! for id, count in result: prms = {'type': type, 'count': count, 'id': id} ! sql(""" update %(type)s set count = count + % (count)d where id = %(id)s """, prms)
  5. Rate Limiting

  6. r = Redis() ! def process_hit(project_id): epoch = time() /

    60 key = ‘{}:{}’.format(project_id, epoch) ! pipe = r.pipeline() pipe.incr(key) pipe.expire(key, 60) result = pipe.execute() ! # return current value return int(result[0])
  7. def request(project_id): result = process_hit(project_id) if result > 20: return

    Response(status=429) return Response(status=200)
  8. Time Series Data

  9. def count_hits_today(project_id): start = time() end = now + DAY_SECONDS

    ! pipe = r.pipeline() for epoch in xrange(now, end, 10): key = ‘{}:{}’.format( project_id, epoch) pipe.get(key) results = pipe.execute() ! # remove non-zero results results = filter(bool, results) # coerce remainder to ints results = map(int, results) # return sum of buckets return sum(results)
  10. Good-enough Locks

  11. from contextlib import contextmanager ! r = Redis() ! @contextmanager

    def lock(key, nowait=True): while not r.setnx(key, '1'): if nowait: raise Locked('try again soon!') sleep(0.01) ! # limit lock time to 10 seconds r.expire(key, 10) ! # do something crazy yield ! # explicitly unlock r.delete(key)
  12. def do_something_crazy(): with lock('crazy'): print 'Hello World!'

  13. Basic Sharding via Nydus

  14. from nydus.db import create_cluster ! redis = create_cluster({ 'backend': 'nydus.db.backends.redis.Redis',

    'hosts': { 0: {'db': 0}, 1: {'db': 1}, }, 'router': 'nydus.db.routers.keyvalue.PartitionRouter', })
  15. def count_hits_today(project_id): start = time() end = now + DAY_SECONDS

    ! keys = [] for epoch in xrange(now, end, 10): key = '{}:{}'.format(project_id, epoch) keys.append(key) ! with redis.map() as conn: results = map(conn.get, keys) ! # remove non-zero results results = filter(bool, results) # coerce remainder to ints results = map(int, results) # return sum of buckets return sum(results)