Upgrade to Pro — share decks privately, control downloads, hide ads and more …

スクレイピングは茨の道/Scraping is a thorny road

6094c91088af5e8ea5094aa750d13570?s=47 cottondesu
December 17, 2018

スクレイピングは茨の道/Scraping is a thorny road

前回LT「Python + Selenium + Beautiful Soup でスクレイピング」の
1.おさらい
2.失敗から学ぶ
3.ソースの修正
4.結論

6094c91088af5e8ea5094aa750d13570?s=128

cottondesu

December 17, 2018
Tweet

Transcript

  1. 2018年12月15日 kanazawa.rb meetup#76 εΫϨΠϐϯά͸ ҵͷಓ

  2. !DPUUPO@EFTV ΏΔͬͱ;ΘͬͱอकΤϯδχΞ ׂ৽ن։ൃɾۀ຿վળ ׂอक ػೳ௥Ճɺػೳमਖ਼ɺόάमਖ਼ FUD

  3. ΞδΣϯμ  લճ-5ͷ͓͞Β͍  લճ-5ࣦഊ͔ΒֶͿ  ιʔεͷमਖ਼  ݁࿦

  4. લճ-5ͷ͓͞Β͍

  5. 1ZUIPO 4FMFOJVN  #FBVUJGVM4PVQͰεΫϨΠϐϯά

  6. εΫϨΠϐϯάͱ͸ʁ

  7. εΫϨΠϐϯά w ΢ΣϒαΠτ͔Β৘ใΛநग़͢Δ ίϯϐϡʔλιϑτ΢ΣΞٕज़ w ݴޠ͸1FSMɺ1ZUIPOɺ3VCZɺ+BWB4DSJQUͰ ϥΠϒϥϦ͋Γ

  8. શମͷߏ੒ਤ

  9. શମͷߏ੒ਤ ϑΥϧμ EJSFOW WFOW QZϑΝΠϧ 4FMFOJVN XFCESJWFS ϩάΠϯϖʔδ $ISPNF

  10. %FNP͕՚ྷʹ Τϥʔ

  11. લճ-5ࣦഊ͔ΒֶͿ

  12. ίʔυ֬ೝ

  13. class main(): options = Options() # ChromeͷύεʢStableνϟωϧͰ--headless͕࢖͑ΔΑ͏ʹͳͬͨΒෆཁͳ͸ͣʣ options.binary_location = '/Applications/Google

    Chrome.app/Contents/MacOS/Google Chrome' # ϔουϨεϞʔυΛ༗ޮʹ͢Δʢ࣍ͷߦΛίϝϯτΞ΢τ͢Δͱը໘͕දࣔ͞ΕΔʣɻ options.add_argument('--headless') # ChromeͷWebDriverΦϒδΣΫτΛ࡞੒͢Δɻ driver = webdriver.Chrome( os.environ["CHROMEDRIVER"], chrome_options=options) # ϩάΠϯαΠτURL driver.get(os.environ["URL"]) # ϩάΠϯID driver.find_element_by_xpath( "//div[@class='sub-content']/dl/dd/div/input").send_keys( os.environ["LOGIN_USER_ID"]) # ϩάΠϯύεϫʔυ username = driver.find_element_by_xpath( "//div[@class='sub-content']/dl/dd[2]/div/input").send_keys( os.environ["LOGIN_USER_PASS"]) # ϩάΠϯϘλϯԡԼ driver.find_element_by_name("login").click() # ޱ࠲؅ཧʹભҠ driver.find_element_by_xpath(“//div[@id='link']/ul/li[10]").click() html = driver.page_source soup = BeautifulSoup(html, 'html.parser')
  14. # อ༗ޱ਺ɺऔಘ୯Ձɺऔಘ୯ՁɺධՁଛӹͷऔಘ item_name = ['อ༗ޱ਺ɹɹ |', 'औಘ୯Ձɹɹ |', 'ج४Ձֹɹɹ |',

    'ධՁଛӹɹɹ |'] # ౤ࢿ৴ୗ໊ͷऔಘ investmentname = [] investments = soup.find_all('td', class_='mbody', colspan="3") for investment in investments: investmentname.append(investment.a.text) inves_num = 0 row_num = 0 various_values = soup.find_all('tr', bgcolor='#eaf4e8', align="right") for various_value in various_values: # ౤ࢿ৴ୗ໊ͷදࣔ print(investmentname[inves_num]) # อ༗ޱ਺ͷදࣔ print(item_name[row_num], various_value.td.text) for other in various_value.td.find_next_siblings("td"): row_num = row_num + 1 # อ༗ޱ਺Ҏ֎ͷදࣔ print(item_name[row_num], other.text) row_num = 0 inves_num = inves_num + 1 # શͯͷΫοΩʔΛ࡟আ driver.delete_all_cookies() driver.quit() if __name__ == "__main__": main()
  15. ໰୊఺

  16. ҟৗऴྃ࣌ʹϓϩηε ͕࢒Γଓ͚Δ ϔουϨεϒϥ΢βͷͨΊΘ͔Γʹ͍͘

  17. ղܾࡦ

  18. ҟৗऴྃ࣌Ͱ΋ϓϩηε Λऴྃͤ͞Δඞཁ͕͋Δ

  19. ιʔεͷमਖ਼

  20. class main(): try: options = Options() # ChromeͷύεʢStableνϟωϧͰ--headless͕࢖͑ΔΑ͏ʹͳͬͨΒෆཁͳ͸ͣʣ options.binary_location =

    '/Applications/Google Chrome.app/Contents/MacOS/Google Chrome' # ϔουϨεϞʔυΛ༗ޮʹ͢Δʢ࣍ͷߦΛίϝϯτΞ΢τ͢Δͱը໘͕දࣔ͞ΕΔʣɻ options.add_argument('--headless') # ChromeͷWebDriverΦϒδΣΫτΛ࡞੒͢Δɻ driver = webdriver.Chrome( os.environ["CHROMEDRIVER"], chrome_options=options) # ϩάΠϯαΠτURL driver.get(os.environ["URL"]) # ϩάΠϯID driver.find_element_by_xpath( "//div[@class='sub-content']/dl/dd/div/input").send_keys( os.environ["LOGIN_USER_ID"]) # ϩάΠϯύεϫʔυ username = driver.find_element_by_xpath( "//div[@class='sub-content']/dl/dd[2]/div/input").send_keys( os.environ["LOGIN_USER_PASS"]) # ϩάΠϯϘλϯԡԼ driver.find_element_by_name("login").click() # ޱ࠲؅ཧʹભҠ driver.find_element_by_xpath(“//div[@id='link']/ul/li[10]").click() html = driver.page_source soup = BeautifulSoup(html, 'html.parser') मਖ਼Օॴ͸੺࿮
  21. # อ༗ޱ਺ɺऔಘ୯Ձɺऔಘ୯ՁɺධՁଛӹͷऔಘ item_name = ['อ༗ޱ਺ɹɹ |', 'औಘ୯Ձɹɹ |', 'ج४Ձֹɹɹ |',

    'ධՁଛӹɹɹ |'] # ౤ࢿ৴ୗ໊ͷऔಘ investmentname = [] investments = soup.find_all('td', class_='mbody', colspan="3") for investment in investments: investmentname.append(investment.a.text) inves_num = 0 row_num = 0 various_values = soup.find_all('tr', bgcolor='#eaf4e8', align="right") for various_value in various_values: # ౤ࢿ৴ୗ໊ͷදࣔ print(investmentname[inves_num]) # อ༗ޱ਺ͷදࣔ print(item_name[row_num], various_value.td.text) for other in various_value.td.find_next_siblings("td"): row_num = row_num + 1 # อ༗ޱ਺Ҏ֎ͷදࣔ print(item_name[row_num], other.text) row_num = 0 inves_num = inves_num + 1 except NoSuchElementException as e: print("seleniumͷૢ࡞தʹΤϥʔ͕ൃੜ͠·ͨ͠ɻ") traceback.print_exc() finally: # શͯͷΫοΩʔΛ࡟আ driver.delete_all_cookies() driver.quit() if __name__ == "__main__": main() मਖ਼Օॴ͸੺࿮
  22. ݁࿦

  23. ݁࿦ w εΫϨΠϐϯά͸ඞͣ੒ޭ͢Δͱ͸ݶΒͳ͍ w ྫ֎ॲཧ USZFYDFQUpOBMMZ ͕ඞཁ w ͍ͭͰ΋ਖ਼ৗऴྃͤ͞Δॲཧ͕ඞཁ