From bd41255361d61d509266c1a7e2c98c858aadd8b8 Mon Sep 17 00:00:00 2001
From: rugantio <costa@caos.space>
Date: Mon, 18 Feb 2019 05:07:38 +0100
Subject: [PATCH] refactoring comments spider

---
 .../__pycache__/comments.cpython-37.pyc       | Bin 4348 -> 4481 bytes
 .../__pycache__/fbcrawl.cpython-37.pyc        | Bin 8337 -> 8337 bytes
 fbcrawl/spiders/comments.py                   |  32 ++++++++++--------
 3 files changed, 18 insertions(+), 14 deletions(-)

diff --git a/fbcrawl/spiders/__pycache__/comments.cpython-37.pyc b/fbcrawl/spiders/__pycache__/comments.cpython-37.pyc
index 4d5fcb9529ff66c6c018706ae293151610af891b..c6999a2a4c545b0782438f938a1baf2ce7a9be44 100644
GIT binary patch
delta 1950
zcmbVNO>7%Q6yDig+v~rz<HU{~H^FJ4uq2J!&;tskN}D)Ug_X2ULW^l-Z9MDL>H4R$
zoBkwgMJN@O5S0?mp%RUd&7mAX>Y-OooVX&vWyOI5mr5MC!GTJ=8OJFthyz~D+j%o@
z-}`3f`^JBcesn}V8x98q_8VKfa%p&1O`_@D*RyY)6R-z+?+Vy!$lFrUhtZ;-`JS@C
zc@4RN^R`jTH*{=}-G|vgh>gR~y1e_t*yL&mM{pGP?MX$?J)wed4ENtvie5Z`<97wf
zmB54WR_+P1Q1nCIBt3?T^dY)vY>ZlEqBrkM;Ps|ashYMytckO_Wz2#<$Ni@W3TcuX
zo6F^9OS#Jz779zHx%0XF?4lc;UA~Z;JD*?56-qPtGmDFE|IEVt%yOZSn_nswau=6#
zi%X02SM<o!2U+6ufjF)*@P24zXx1b8y*L#9#6fM*5j$vBxa!b9#OW<taL^jUVoKOX
zj<*9-NU)VPzvFd;wSXhTY_J!B130)R<IsKSeE|tK4qt+AyXkh2HHE`1-|dg>5Fhh(
z&^9Y#UlChMOM+aUwil~ypQEsh`*p(hmbxuC5{}##TYd<NIR3RLj-C=6f4dL&VYwrs
zZXdS#pby)<-;d!$Hyw}a?+6^&w=V$u;&zci!h?L(`BuOQfS2r`w$u?$w_tT`*$HAL
zC3FyYU8E^M8tkBh-C*MbSj0nB)LZ`!x>0zNmp?_h^kyqlH!R!0nR266GwOC`I@8ow
zjm&1tm8<ott;s}zQy@WRl3-l9V(P1!=qhF1G*|TUb*gzko8CACsfe+<ZtCQpxgcDw
z0zX~FGEAdvH;5~ndVQ7nSX#z*`aw87MA)Hks1w<&zOq8HwsG5jL*u1s+;>hTj2Bwm
zl`NZ(5xVKSm}anCzfoVW8AJzair^EOnNj+aFZHU9ap|URt{bl31Z|W!h(J>(mQiBs
z)IuaiRVAHfKwPh0w{984mEc|UG<Eyhq>HNbjFP^TU^%%NWW5t$*lt*-!3e$tiXs)B
zBpN|pBqI@;f?{F_$zl>EU?#NxK$H~PkB%WPeXNWPA5i@lfDfz?FiHtnye{q_^7WJ7
z6%Rs~KJy=);&X7^IL>=&svDhvkwKt$^1jSiRG;n5wtR5-!FC9zXep354o;VL0%o5K
z4GO@CbXOBQsDARGbR0^H<Y)S2U=w{uj|4}T_ke3*$AkMh*Sxk0gl9U7;(`AS>JS5U
z_`g8mVYrG$=#RnoyoX?$NtzxF)vmt6x*29hnPDf#Idko;tlr}c<E$$e78aH`yEgiO
zQ@DEb!xLAty^w$B8vQNw%~s|t&^l9Ttk>~)rq&?FOCqrdyYZf4UW7@2E2AeB4)s7U
znnZ?KoYExK2C3S2$OyO_Q}Zv#&dR_gd&|sJKywzIP@}_5(r6lli*X9n*yX;+a{>2N
z_1(#Nt)J&2M<C(fna|u~%uFzoWro8!P9Lk|hZ%J>g~ZtKBp6pFjfPF$X8uWL-l5Ob
z^uSAmAA^}0Iv$zoW9(LVONaqryqDgJycT(R?5jv+i(TkkDTICPhD)WgsasarFwIh_
zD<)P&LnO{3xv)IYV~1xAu#O~yfD{zy7L-DgGDcVg;Md?)4DbIpF>M=!??G{}vcmUr
sf_@&2<|Q818)EXey7vQ1^Blu-X}ij)S_7|}#%XrO7Q>=|j^UC10wYMp?EnA(

delta 1849
zcmb_dO>7%Q6yDig?~d2M`E%^FX|@#FSP3zzKq^%QRcTTxMF~x#f>uS*CbM?!*xq!!
zp-sBRg;wgNsS?c%aj+5Mgb-4>AoaqX3lgWja-k9z?i_e;97pBv#*X&Qy!XcM&D-~V
z<4;cgaWZ`=m9i9g&KR|uKOa9x=gH#1nbJH{SgdX~s6&ZzMPUXrA1ciBj0c(%XXLt4
zq|fnJOGOe{R|BuH=)270pd}W3-dOQ&Z+ora;C{(U;Wl_$s|VlD%&|-*StU;Fky0is
z%Z46O$6&*3<e}o2Y?O_`Odlz_;>00EPUMI)GQ2Q9e|c^GgZGwKR@WA;&M#iRE_|{g
zPOD$VG7xxF{G`sN6R^TMHZax4BB4#B?uVqShH8&oRzBq7tah;z8a>h@L<wju9vY!i
zvzQg?A*m(cB}^?SEWwhW>nydW-%yBh`=$4lL6CjY(pkD~-gTtKlwlX!T96D<50p@2
znLV{lA!a(HwG7K%P(s>;3Yp$hNq^meLXQ+E_HaGaQ)II5{*r8WkOmU(-o1$$%gLlm
z?RXdmZM;XiT2E<P;L}5k(Sp(=peeOj*I@as8Coxt3#I13HOy32s1i2OuW+B-o__j`
z{+L`VEJ4FOW^ee7hSv;iH?X(dif7+zuSR;cSq+Lh#}CZ0p*#m7(#v(XQq&^4;nwTx
z?#5=6V16@LD_84*$3@5ZY^tcjj{zPLZFzNXBk*~o*WG4?$1zG$+%Ts`v30xYxl+kp
zU+1O3yBkaw#S`<&)N=x!qQ;No-eV{x(J&Het$_1M@r!vaD^H`MFNVSv%d{X`G^Zta
zRy?N1I+!Rj+-B>J$0H49C${AVx6Vdd)4Suun~-h|3w0>ya>Yk6k`!aODuEb>w{;$b
zV&+Jeq+#UAFfm}LL?@ICt4WyAlRO-0Ef2bhTZai1ug7O5j{y1`oERXdFanTMH`RT@
zzkM3UV}z8&r`G8X8<Sq>D9g!Jy3pT04wgIsw6GUL%rqAU-nI$NO>~oNqB}IO9d<um
z%Y;BK+5B<o!zjnb6@bC<0SubwqzBKF+l;CED*shXC+?9S#g~bb<O}h{Ds;vm2p;;s
zk2U%K9*doTTVQJUUCYKtILUVs6VL))0I}<d-;Gi6EV-IJfwn0WHj22gQ=>`z){(xl
zyu1o9yO<i!{Ts}9N;Fc>DwCH0ymr~&Zn9ZB;5*f3#SVPC;@gO7jxbz(&SXGoK=cpj
zAP;oJ39w3_B4i`F#j8HA20IeByXVASdZu&BZ+Mbf=Yd-MBu4rN!ViPI3C=$F5%){Q
z5gCV{Lh}odM|Bp(t00`X%h=jSZhhP9Gj$F<FQYh(Laby?Tth?`DbJ$eH4u@`eLvuD
zpnev`IS@`9YwdHxk~HDoR(<F1?3jVAZ;Kx?=ZBCARK7XY0(wx9%)XLAa-e-+yO=E-
zcy05N_&S?kKjK=E@DVgi${Dai;<oUJRMUwL5iOF`k{X@n=q*~j2$Pd)ZE$yMXRa>)
zf8=M05{I^1L9oR&ksq2|(&gUrI&8jEpTire)xR(>)(a&au|A>~8a~^udv9U)TZkqK
I6qiH$6CC!Gga7~l

diff --git a/fbcrawl/spiders/__pycache__/fbcrawl.cpython-37.pyc b/fbcrawl/spiders/__pycache__/fbcrawl.cpython-37.pyc
index 098523eeec4eb43473f5d381a1d7356438e87900..a0153e469d24acedeb3d7a08cb0888ee55cd985a 100644
GIT binary patch
delta 46
zcmbQ}IMI>YiI<m)0SIo2XKmy@CBb-W^EHX-ER12B^JLdCGKNo<m7fPB_sCxb068lT
A$p8QV

delta 46
zcmbQ}IMI>YiI<m)0SH=7Wp3m?CBYcA`I^LZ7RJ!cd9v#m8N(*a%FhFmd*rVI05{bR
AqW}N^

diff --git a/fbcrawl/spiders/comments.py b/fbcrawl/spiders/comments.py
index 728010c..41bccd5 100644
--- a/fbcrawl/spiders/comments.py
+++ b/fbcrawl/spiders/comments.py
@@ -1,7 +1,6 @@
 import scrapy
 
 from scrapy.loader import ItemLoader
-from scrapy.http import FormRequest
 from fbcrawl.spiders.fbcrawl import FacebookSpider
 from fbcrawl.items import CommentsItem
 
@@ -14,7 +13,9 @@ class CommentsSpider(FacebookSpider):
     custom_settings = {
         'FEED_EXPORT_FIELDS': ['source','reply_to','date','text', \
                                'reactions','likes','ahah','love','wow', \
-                               'sigh','grrr','url']
+                               'sigh','grrr','url'],
+        'DUPEFILTER_CLASS' : 'scrapy.dupefilters.BaseDupeFilter',
+        'CONCURRENT_REQUESTS':1, 
     }
 
     def __init__(self, *args, **kwargs):
@@ -32,7 +33,7 @@ class CommentsSpider(FacebookSpider):
             source = reply.xpath('.//h3/a/text()').extract()
             answer = reply.xpath('.//a[contains(@href,"repl")]/@href').extract()
             ans = response.urljoin(answer[::-1][0])
-            self.logger.info('Nested comment at page {}'.format(ans))
+            self.logger.info('{} nested comment @ page {}'.format(str(response.meta['index']),ans))
             yield scrapy.Request(ans,
                                  callback=self.parse_reply,
                                  meta={'reply_to':source,
@@ -42,16 +43,19 @@ class CommentsSpider(FacebookSpider):
         #loads regular comments     
         if not response.xpath(path):
             path2 = './/div[string-length(@class) = 2 and count(@id)=1 and contains("0123456789", substring(@id,1,1)) and not(.//div[contains(@id,"comment_replies")])]'
-            for reply in response.xpath(path2):
+            
+            for i,reply in enumerate(response.xpath(path2)):
+                self.logger.info('{} regular comment @ page {}'.format(i,response.url))
                 new = ItemLoader(item=CommentsItem(),selector=reply)
                 new.context['lang'] = self.lang           
                 new.add_xpath('source','.//h3/a/text()')  
                 new.add_xpath('text','.//div[h3]/div[1]//text()')
                 new.add_xpath('date','.//abbr/text()')
+                new.add_value('url',response.url)
                 yield new.load_item()
-#            
-#        #previous comments
-        if not response.xpath(path) and not response.xpath(path2):
+            
+        #previous comments
+        if not response.xpath(path):
             for next_page in response.xpath('.//div[contains(@id,"see_next")]'):
                 new_page = next_page.xpath('.//@href').extract()
                 new_page = response.urljoin(new_page[0])
@@ -59,7 +63,7 @@ class CommentsSpider(FacebookSpider):
                 yield scrapy.Request(new_page,
                                      callback=self.parse_page,
                                      meta={'index':1})        
-#        
+        
     def parse_reply(self,response):
         '''
         parse reply to comments, root comment is added if flag
@@ -88,7 +92,7 @@ class CommentsSpider(FacebookSpider):
                 
             back = response.xpath('//div[contains(@id,"comment_replies_more_1")]/a/@href').extract()
             if back:
-                self.logger.info('Back found, trying to go back')
+                self.logger.info('Back found, more nested comments')
                 back_page = response.urljoin(back[0])
                 yield scrapy.Request(back_page, 
                                      callback=self.parse_reply,
@@ -99,8 +103,8 @@ class CommentsSpider(FacebookSpider):
                                            'index':response.meta['index']})
             else:
                 next_reply = response.meta['url']
-                self.logger.info('Nested comments crawl finished, heading to home page: {}'.format(response.meta['url']))
-                yield scrapy.Request(next_reply, dont_filter=True,
+                self.logger.info('Nested comments crawl finished, heading to proper page: {}'.format(response.meta['url']))
+                yield scrapy.Request(next_reply,
                                      callback=self.parse_page,
                                      meta={'index':response.meta['index']+1})
                 
@@ -117,7 +121,7 @@ class CommentsSpider(FacebookSpider):
                 yield new.load_item()
             #keep going backwards
             back = response.xpath('//div[contains(@id,"comment_replies_more_1")]/a/@href').extract()
-            self.logger.info('Back found, trying to go back')
+            self.logger.info('Back found, more nested comments')
             if back:
                 back_page = response.urljoin(back[0])
                 yield scrapy.Request(back_page, 
@@ -130,6 +134,6 @@ class CommentsSpider(FacebookSpider):
             else:
                 next_reply = response.meta['url']
                 self.logger.info('Nested comments crawl finished, heading to home page: {}'.format(response.meta['url']))
-                yield scrapy.Request(next_reply, dont_filter=True,
+                yield scrapy.Request(next_reply,
                                      callback=self.parse_page,
-                                     meta={'index':response.meta['index']+1})
\ No newline at end of file
+                                     meta={'index':response.meta['index']+1})