Slide 33
Slide 33 text
from operator import itemgetter
from transducer.eager import transduce
from transducer.transducers import grouping, mapping, batching
from transducer.functional import compose
from transducer.reducers import Appending
normprice = lambda i: int(float(i.replace(",", ".")) * 100)
COLUMNS = (
("int_id", str), ("num", str), ("id", str),
("sku", int), ("qty", float), ("ppi_price", normprice),
("price", normprice), ("discount_price", normprice),
("created_at", str), ("location", str))
class AppendToDatabase(Appending):
def step(self, result, item):
print ("Here we going to save into database: {}".format(len(item)))
pprint.pprint(transduce(
transducer=compose(
mapping(lambda l: l.split(";")),
mapping(lambda c: [f(v) for f, v in zip(map(itemgetter(1), COLUMNS), c)])
,
mapping(lambda c: dict(zip(map(itemgetter(0), COLUMNS), c))),
grouping(itemgetter("id")),
batching(100)
),
reducer=AppendToDatabase(),
iterable=gather_data()))