Save 37% off PRO during our Black Friday Sale! »

Des Boucles Aux Transducers - Meetup CraftingSw Paris 2016

Des Boucles Aux Transducers - Meetup CraftingSw Paris 2016

Les boucles, tout le monde les utilise pour transformer de large collection.

Et si vous les abandonniez en gagnant par la même occasion en clarté ? Et si vous pouviez appliquer les mêmes transformations sur des ensembles infinis tout en améliorant drastiquement les performances ?

Par refactoring successif, nous passerons des boucles aux filtres et pipelines pour clôturer sur les transducers.

Une présentation qui mélange transformation de données et programmation fonctionnelle (tout en restant ouverte aux débutants).

Beb422437c1dfb5366f197919e41ac50?s=128

Arnaud LEMAIRE
PRO

November 30, 2016
Tweet

Transcript

  1. DES BOUCLES AUX TRANSDUCTEURS @LILOBASE - ARPINUM.FR

  2. IL ÉTAIT UNE COLLECTION DE DONNÉE invoices = (
 dict(


    due_date=datetime(2016, 10, 22),
 amount_excl_vat=36,
 quantity=1,
 vat_rate=20,
 paid=False
 ),
 dict(
 due_date=datetime(2016, 9, 18),
 amount_excl_vat=28,
 quantity=1,
 vat_rate=10,
 paid=True
 ),
 dict(
 due_date=datetime(2016, 10, 23),
 amount_excl_vat=17,
 quantity=2,
 vat_rate=5,
 paid=False
 )
 )
  3. EXEMPLE DE TRAITEMENT IMPÉRATIF def get_amount_for_month(invoices, month):
 total_amount = 0


    for invoice in invoices:
 if (invoice['due_date'].month != month):
 continue
 
 total_amount += invoice['amount_excl_vat']\
 + (invoice['amount_excl_vat']
 / 100.0
 * invoice['vat_rate']) \
 * invoice['quantity']
 
 return total_amount
  4. PIPELINES MAP, FILTER, REDUCE…

  5. def get_amount_for_month(invoices, month):
 total_amount = 0
 for invoice in invoices:


    if (invoice['due_date'].month != month):
 continue
 
 total_amount += invoice['amount_excl_vat']\
 + (invoice['amount_excl_vat']
 / 100.0
 * invoice['vat_rate']) \
 * invoice['quantity']
 
 return total_amount IDENTIFICATION DES ÉTAPES DU PIPELINE filter = tri 1 1 reduce = accumulateur 2 2 map = traitements sur chaque élément de la collection 3 3
  6. REFACTORONS POUR UTILISER DES PIPELINES def get_amount_for_month(invoices, month):
 invoices_of_the_month =

    filter(
 lambda invoice: invoice['due_date'].month == month,
 invoices
 )
 
 incl_vat_amount = map(
 lambda invoice: invoice['amount_excl_vat']\
 + (invoice['amount_excl_vat']
 / 100.0
 * invoice['vat_rate']) \
 * invoice['quantity'],
 invoices_of_the_month
 )
 
 return reduce(
 lambda carry, x: carry + x,
 incl_vat_amount
 )
  7. EXTRACTION DES FONCTIONS def invoice_is_due_for(month):
 return lambda invoice: \ invoice['due_date'].month

    == month def including_vat(invoice):
 return invoice[‘amount_excl_vat'] \
 + (invoice['amount_excl_vat']
 / 100.0
 * invoice['vat_rate']) \
 * invoice['quantity']
 
 
 def sum_invoice_amount(carry, x):
 return carry + x Closure = pré-contextualisation d’une fonction 1 1
  8. VERSION REFACTORÉE def get_amount_for_month(invoices, month):
 invoice_of_the_month = \
 filter(invoice_is_due_for(month), invoices)


    
 including_vat_amount = \
 map(including_vat, invoice_of_the_month)
 
 return reduce(sum_invoice_amount, including_vat_amount)
  9. COLLECTION PIPELINE Les comportements sont isolées et sans effet de

    bord Les comportements peuvent être réutilisés Le code est plus modulaire et résilient
  10. MAIS… L’ensemble des éléments de la collection est parcouru à

    chaque étape de transformation Chaque étape de transformation crée une collection intermédiaire Chaque étape supplémentaire augmente le temps de calcul et la mémoire consommée
  11. AVEC DES DESSINS C’EST PLUS FACILE COLLECTION COLLECTION COLLECTION COLLECTION

    Invoices of the month VAT included amount sum Invoices
  12. PARLONS DE REDUCE REDUCE…

  13. REDUCE PEUT ACCUMULER DU NON SCALAIRES def letter_count(sentence):
 
 def

    acc_letter(carry, word):
 if(not carry.has_key(word)):
 carry[word] = 1
 else:
 carry[word] += 1
 
 return carry
 
 return reduce(acc_letter, sentence, {})
  14. MAP OU FILTER PEUVENT ÊTRE EXPRIMÉS VIA REDUCE def reduced_map(callable,

    collection):
 
 def map_reducer(carry, item):
 carry.append(callable(item))
 return carry
 
 return reduce(map_reducer, collection, [])
 
 def reduced_filter(callable, collection):
 
 def filter_reducer(carry, item):
 if callable(item):
 carry.append(item)
 return carry
 
 return reduce(filter_reducer, collection, [])
  15. ON PEUT DONC RÉÉCRIRE NOTRE FONCTION PRÉCÉDENTE SOUS FORME D’UN

    REDUCER def get_amount_for_month_reduced(invoices, month):
 def reducer(carry, invoice):
 if invoice['due_date'].month == month:
 carry += invoice['amount_excl_vat'] \
 + (invoice['amount_excl_vat'] / 100
 * invoice['vat_rate']) \
 * invoice['quantity']
 return carry
 
 return reduce(reducer, invoices, 0)
  16. AVEC DES DESSINS C’EST PLUS FACILE COLLECTION COLLECTION Sum Invoices

  17. MAIS… C’est complètement illisible

  18. TRANSDUCERS MAPPING, FILTERING, REDUCING…

  19. SORTONS L’INSTRUCTION REDUCE def mapper(callable):
 
 def map_reducer(carry, item):
 carry.append(callable(item))


    return carry
 
 return map_reducer
 
 def filterer(callable):
 
 def filter_reducer(carry, item):
 if callable(item):
 carry.append(item)
 return carry
 
 return filter_reducer
  20. None
  21. UTILISATION def paid_invoice(invoice):
 return invoice['paid']
 
 
 
 reduce(filterer(paid_invoice), invoices,

    [])
  22. DÉPLAÇONS APPEND def appender(carry, item):
 carry.append(item)
 return carry

  23. ET NOS RÉDUCERS DEVIENNENT TRANSDUCERS def mapping(callable):
 def transducer(next_reducer):
 


    def map_reducer(carry, item):
 return next_reducer(carry, callable(item))
 
 return map_reducer
 
 return transducer
 
 
 def filtering(callable):
 def transducer(next_reducer):
 
 def filter_reducer(carry, item):
 if callable(item):
 return next_reducer(carry, item)
 else:
 return carry
 
 return filter_reducer
 
 return transducer
  24. None
  25. ET VOICI LES TRANSDUCTEURS À L’ACTION def adder(carry, item):
 return

    carry + item
 
 
 
 reduce(
 filtering(invoice_is_due_for(10))(
 mapping(including_vat)(
 adder
 )
 ), 
 invoices, 
 0
 )
  26. TRANSDUCERS L’ensemble des traitements est composé 
 en une fonction

    Chaque élément de la collection parcourt l’ensemble des transformations en une fois Le nombre d’itération ne dépend plus du nombre de transformation
  27. TRANSDUCERS Gains en consommation mémoire et temps de calcul Peut

    traiter des ensembles infinis Plus besoin de créer des représentations intermédiaires de la collection
  28. AVEC DES DESSINS C’EST PLUS FACILE COLLECTION COLLECTION sum Invoices

    }
  29. Les traitements se font successivement sur la collection L’ensemble des

    transformations est composé en une fonction pipelines « classiques » transducers
  30. SUCRE SYNTAXIQUE COMPOSE, MAPITY, FILTERITY

  31. COMPOSE def compose2(f, g):
 return lambda *a, **kw: f(g(*a, **kw))


    
 
 def compose(*fs):
 return reduce(compose2, fs)
 
 
 def get_amount_for_month(invoices, month):
 return reduce(
 compose(
 filtering(invoice_is_due_for(month)),
 mapping(including_vat),
 )(adder),
 invoices,
 0
 )
  32. None
  33. AMÉLIORONS L’API DES TRANSDUCERS def mapity(callable, collection = None):
 if

    collection is None:
 return mapping(callable)
 else:
 return map(callable, collection)
 
 
 def filterity(callable, collection = None):
 if collection is None:
 return filtering(callable)
 else:
 return filter(callable, collection)
  34. AMÉLIORONS L’API DES TRANSDUCERS, SUITE… filterity(
 paid_invoice, 
 mapity(
 including_vat,

    
 invoices
 )
 )
 
 compose(
 filterity(paid_invoice),
 mapity(including_vat)
 )(appender) Génération d’un transducer Utilisation avec les pipelines « classiques »
  35. STATEFULLNESS PARTITION_BY, BATCH, FIRST

  36. EN OOP, EN PHP ET AVEC UN ÉTAT ! class

    Batching implements Reducer
 {
 protected $next_reducer;
 protected $batch_size;
 protected $current_batch = [];
 
 public function __construct(Reducer $next_reducer, Integer $batch_size)
 {
 $this->batch_size = $batch_size;
 $this->next_reducer = $next_reducer;
 }
 
 public function init()
 {
 return $this->next_reducer->init();
 }
 
 public function step($result, $current)
 {
 $this->current_batch[] = $current;
 if (count($this->current_batch) >= $this->batch_size) {
 $batch = $this->current_batch;
 $this->current_batch = [];
 
 return $this->next_reducer->step($result, $batch);
 }
 
 return $result; 
 }
 
 public function complete($result)
 {
 if (count($this->current_batch) > 0) {
 $result = $this->next_reducer->step($result, $this->current_batch);
 }
 
 return $this->next_reducer->complete($result);
 }
 }
  37. NE BLOQUEZ PAS LE PIPELINE ! class Partitioning extends Reducer

    {
 
 private $predicate;
 private $next_reducer;
 private $partions = [[], []];
 
 public function __construct(Reducer $next_reducer, Callable $predicate) {
 $this->next_reducer = $next_reducer;
 $this->predicate = $predicate;
 } public function init() {
 return $this->next_reducer->init();
 }
 
 public function step($result, $current) {
 if($this->predicate->__invoke($current))
 $this->partions[0] = $current;
 else
 $this->partions[1] = $current;
 }
 
 public function complete($result) {
 return $this->partions;
 }
 }
  38. NE BLOQUEZ PAS LE PIPELINE class Partitioning extends Reducer {


    private $predicate;
 private $next_reducer;
 
 public function __construct(Reducer $next_reducer, Callable $predicate) {
 $this->next_reducer = $next_reducer;
 $this->predicate = $predicate;
 }
 
 public function init() {
 return [[], []];
 }
 
 public function step($result, $current) {
 if ($this->callback($current)) {
 $result[0][] = $current;
 } else {
 $result[1][] = $current;
 }
 
 return $result;
 }
 
 public function complete($result) {
 return $result;
 }
 }
  39. EARLY TERMINATION namespace Fp\Reducer;
 
 class First implements Reducer
 {


    use Mixin\Stateless;
 use Mixin\ConstructWithCallback;
 
 public function step($result, $current)
 {
 if ($this->callback->__invoke($current)) {
 return new Reduced($this->next_reducer->step($result, $current));
 }
 
 return $result;
 }
 
 }

  40. TRANSDUCE function transduce(callable $transducer, Reducer $reducer, Iterable $iterable, $init =

    null)
 {
 $internal_reducer = $transducer($reducer);
 
 $accumulator = (is_null($init)) ? $internal_reducer->init() : $init;
 foreach ($iterable as $current) {
 $accumulator = $internal_reducer->step($accumulator, $current);
 
 //early termination
 if ($accumulator instanceof Reduced) {
 $accumulator = $accumulator->value();
 break;
 }
 }
 
 return $internal_reducer->complete($accumulator);
 }
  41. USAGE Fp\transduce(
 Fp\compose(
 Fp\mapping(square_makker()),
 Fp\batching(3)
 ),
 Fp\appending(), range(1, 6)); Fp\transduce(


    Fp\compose(
 Fp\mapping(square_makker()),
 Fp\first(function ($x) { return $x > 6; })
 ),
 Fp\single_result(), range(1, 6)); Fp\transduce(
 Fp\compose(
 Fp\mapping(square_makker()),
 Fp\filtering(is_even_makker()),
 Fp\enumerating()
 ),
 Fp\appending(), range(1, 6));
  42. @LILOBASE MERCI !