Django works great for single database webapps, but did you know they give us the tools to create webapps with sharded data right out of the box? This talk will go over how we leveraged Django's features to add sharding to our webapp infrastructure.
def db_for_read(model, **hints): # Given a model class `model`, return the alias that maps to the # the database connection for reading, or None if there is no opinion. pass def db_for_write(model, **hints): # Given a model class `model`, return the alias that maps to the # the database connection for writing, or None if there is no opinion. pass def allow_relation(obj1, obj2, **hints): # Return True if the relation between `obj1` and `obj2` should be # explicitly allowed, return False if the relation between `obj1` # and `obj2` should be strictly disallowed, or return None if the # router should have no opinion. pass def allow_syncdb(db, model): # Given a database alias `db` and a Django model `model` return True # if the model should be synchronized for the given alias, return False # if the router should not be synchronized, or return None if the router # has no opinion. pass 25 Thursday, 5 September, 13
self.db_name): raise AttributeError('missing class variables `app_name` or `db_name`') def db_for_read(self, model, **hints): if model._meta.app_label == self.app_name: return self.db_name return None def db_for_write(self, model, **hints): if model._meta.app_label == self.app_name: return self.db_name return None def allow_relation(self, obj1, obj2, **hints): # only allow joins between objects in the same app if obj1._meta.app_label == self.app_name and obj2._meta.app_label == self.app_name: return True elif obj1._meta.app_label == self.app_name or obj2._meta.app_label == self.app_name: return False else: # neither of the models are from the app - we have no opinion. return None def allow_syncdb(self, db, model): if model._meta.app_label in ['south']: return True if db == self.db_name: return model._meta.app_label == self.app_name return None App-based feature partitioning database router. 33 33 Thursday, 5 September, 13
self.db_name): raise AttributeError('missing class variables `app_name` or `db_name`') def db_for_read(self, model, **hints): if model._meta.app_label == self.app_name: return self.db_name return None def db_for_write(self, model, **hints): if model._meta.app_label == self.app_name: return self.db_name return None def allow_relation(self, obj1, obj2, **hints): # only allow joins between objects in the same app if obj1._meta.app_label == self.app_name and obj2._meta.app_label == self.app_name: return True elif obj1._meta.app_label == self.app_name or obj2._meta.app_label == self.app_name: return False else: # neither of the models are from the app - we have no opinion. return None def allow_syncdb(self, db, model): if model._meta.app_label in ['south']: return True if db == self.db_name: return model._meta.app_label == self.app_name return None App-based feature partitioning database router. We will come back to this later 33 33 Thursday, 5 September, 13
obj1, obj2, **hints): # only allow joins between objects in the same app if obj1._meta.app_label == self.app_name and obj2._meta.app_label == self.app_name: return True elif obj1._meta.app_label == self.app_name or obj2._meta.app_label == self.app_name: return False else: # neither of the models are from the app - we have no opinion. return None def allow_syncdb(self, db, model): if model._meta.app_label in ['south']: return True if db == self.db_name: return model._meta.app_label == self.app_name return None 36 Thursday, 5 September, 13
obj1, obj2, **hints): # only allow joins between objects in the same app if obj1._meta.app_label == self.app_name and obj2._meta.app_label == self.app_name: return True elif obj1._meta.app_label == self.app_name or obj2._meta.app_label == self.app_name: return False else: # neither of the models are from the app - we have no opinion. return None def allow_syncdb(self, db, model): if model._meta.app_label in ['south']: return True if db == self.db_name: return model._meta.app_label == self.app_name return None 37 Thursday, 5 September, 13
config(default=None, **kwargs): """ Returns a database dictionary from a URI. Updates the resulting database dictionary with the `kwargs` passed in. """ database_config = dj_database_url.config(default=default) database_config.update(kwargs) return database_config 56 Thursday, 5 September, 13
models.IntegerField() ... def get_shard(self): """ Returns the database alias that the data should reside on. """ if self._state.db: return self._state.db sharding_key = ShardingKey.objects.get(pk=self.sharding_key_pk) return sharding_key.shard_database_alias 59 Thursday, 5 September, 13
= None def db_for_read(self, model, **hints): if model._meta.app_label in self.app_list: instance = hints.get('instance') if instance: return instance.get_shard() return None def db_for_write(self, model, **hints): if model._meta.app_label in self.app_list: instance = hints.get('instance') if instance: return instance.get_shard() return None def allow_syncdb(self, db, model): if model._meta.app_label in ['south']: return True this_app = (model._meta.app_label in self.app_list) _db = db.startswith(self.db_names_prefix) if this_app: return _db if _db: return False return None NOTE: We did not define an allow_relation() method because we deferred to the default behaviour. 64 Thursday, 5 September, 13
ashchristopher
shoheimitani
.jpeg){kind=avatar}
arthur1
chencmd
dropout009
kbaba1001
saitolume
datanalyticslabo
nashiusagi
shukob
.jpeg){kind=avatar}
ostk0069
iwatatomoya
kimura_m_29
brad_frost
eileencodes
dougneiner
kneath
jonrohan
pedronauck
chrislema
sachag
tammielis
kastner
chrisshort
mraible
![Database Sharding with Django Ash Christopher [email protected] @ashchristopher 1 DjangoCon](https://files.speakerdeck.com/presentations/bd798ae0f8670130b555127c538d0e80/slide_0.jpg){kind=link}
