added experimental support for languages en, es, fr, it, pt

fbcrawl/items.py

@@ -9,7 +9,9 @@ import scrapy
 from scrapy.loader.processors import TakeFirst, Join, MapCompose
 from datetime import datetime, timedelta
 
-def parse_date(init_date):
+def parse_date(init_date,loader_context):
+    lang = loader_context['lang']
+    if lang == 'it':
         mesi = {
         'gennaio':1,
         'febbraio':2,
@@ -106,6 +108,8 @@ def parse_date(init_date):
         else:
         #date item parser fail. datetime format unknown, check xpath selector or change the language of the interface'
             return init_date
+    else:
+        return init_date
     date = datetime(year,month,day)
     return date.date()
 

fbcrawl/spiders/fbcrawl.py

@@ -11,7 +11,7 @@ class FacebookSpider(scrapy.Spider):
     """    
     name = "fb"
 
-    def __init__(self, email='', password='', page='', year=2018, **kwargs):
+    def __init__(self, email='', password='', page='', year=2018, lang='', **kwargs):
         super(FacebookSpider, self).__init__(**kwargs)
         
         self.year = int(year)    #arguments are passed as strings
@@ -27,8 +27,19 @@ class FacebookSpider(scrapy.Spider):
         else:
             self.page = page
             
-        self.start_urls = ['https://mbasic.facebook.com']    
+        if not(lang):
+            self.logger.info('Language attribute not provided, assuming "en"')
+            self.logger.info('Currently supported languages are: "en", "es", "fr", "it", "pt"')                             
+            self.lang = 'en'
+        elif lang == 'en'  or lang == 'es' or lang == 'fr' or lang == 'it' or lang == 'pt':
+            self.lang = lang
+        else:
+            self.logger.info('Lang:{} not currently supported'.format(lang))                             
+            self.logger.info('Currently supported languages are: "en", "es", "fr", "it", "pt"')                             
+            self.logger.info('Change your interface lang from facebook and try again')     
+            return                        
 
+        self.start_urls = ['https://mbasic.facebook.com']    
 
     def parse(self, response):
         return FormRequest.from_response(
@@ -59,29 +70,24 @@ class FacebookSpider(scrapy.Spider):
             return FormRequest.from_response(
                 response,
                 formdata={'approvals_code': self.code},
-                callback=self.parse_home,
+                callback=self.parse_home)
-            )
+        elif response.xpath("//div/a[contains(@href,'save-device')]"):
-        elif response.xpath("//div/input[@value='Ok' and @type='submit']"):
+#        elif response.xpath("//div/input[@value='Ok' and @type='submit']"):
             # Handle 'Save Browser' checkpoint.
             return FormRequest.from_response(
                 response,
                 formdata={'name_action_selected': 'dont_save'},
-                callback=self.parse_home,
+                callback=self.parse_home)
-                dont_filter=True,
-            )
         elif response.css('button#checkpointSubmitButton'):
             # Handle 'Someone tried to log into your account' warning.
             return FormRequest.from_response(
-                response, callback=self.parse_home, dont_filter=True,)
+                response, callback=self.parse_home)
         # Else go to the page requested.
-        if self.page.find('.facebook.com/') != -1:
+        if self.page.find('https://www.facebook.com/') != -1:
-            self.page = self.page[28:]
+            self.page = self.page[25:]
         href = response.urljoin(self.page)
         self.logger.info('Parse function called on %s', href)
-        return scrapy.Request(
+        return scrapy.Request(url=href,callback=self.parse_page)
-            url=href,
-            callback=self.parse_page,
-        )
 
     def parse_page(self, response):
         #select all posts
@@ -97,29 +103,35 @@ class FacebookSpider(scrapy.Spider):
             yield scrapy.Request(temp_post, self.parse_post, meta={'item':new})       
 
         #load following page
-#        next_page = response.xpath('//*[@id="structured_composer_async_container"]/div[2]/a/@href')
+        #next_page = response.xpath('//*[@id="structured_composer_async_container"]/div[2]/a/@href')
-        next_page = response.xpath("//div[2]/a[contains(@href,'timestart=') and not(contains(text(),'ece')) and not(contains(text(),number()))]/@href").extract()      
+        next_page = response.xpath("//div[2]/a[contains(@href,'timestart=') and not(contains(text(),'ent')) and not(contains(text(),number()))]/@href").extract()      
         if len(next_page) == 0: 
-            if response.meta['flag'] == 3 and self.year <= 2015:
+            if response.meta['flag'] == 4 and self.year <= 2015:
-                self.logger.info('2015 reached, flag = {}'.format(response.meta['flag']))
+                self.logger.info('2014 reached, flag = 5')
+                next_page = response.xpath("//div/a[contains(@href,'time') and contains(text(),'2015')]/@href").extract()
+                self.logger.info('next_page = {}'.format(next_page[0]))
+                new_page = response.urljoin(next_page[0])
+                yield scrapy.Request(new_page, callback=self.parse_page, meta={'flag':5}) 
+            elif response.meta['flag'] == 3 and self.year <= 2015:
+                self.logger.info('2015 reached, flag = 4')
                 next_page = response.xpath("//div/a[contains(@href,'time') and contains(text(),'2015')]/@href").extract()
                 self.logger.info('next_page = {}'.format(next_page[0]))
                 new_page = response.urljoin(next_page[0])
                 yield scrapy.Request(new_page, callback=self.parse_page, meta={'flag':4}) 
             elif response.meta['flag'] == 2 and self.year <= 2016:
-                self.logger.info('2016 reached, flag = {}'.format(response.meta['flag']))                
+                self.logger.info('2016 reached, flag = 3')                
                 next_page = response.xpath("//div/a[contains(@href,'time') and contains(text(),'2016')]/@href").extract()
                 self.logger.info('next_page = {}'.format(next_page[0]))
                 new_page = response.urljoin(next_page[0])
                 yield scrapy.Request(new_page, callback=self.parse_page, meta={'flag':3}) 
             elif response.meta['flag'] == 1 and self.year <= 2017:            
-                self.logger.info('2017 reached, flag = {}'.format(response.meta['flag']))                
+                self.logger.info('2017 reached, flag = 2')          
                 next_page = response.xpath("//div/a[contains(@href,'time') and contains(text(),'2017')]/@href").extract()
                 self.logger.info('next_page = {}'.format(next_page[0]))
                 new_page = response.urljoin(next_page[0])
                 yield scrapy.Request(new_page, callback=self.parse_page, meta={'flag':2})      
             elif response.meta['flag'] == 0 and self.year <= 2018:                      
-                self.logger.info('2018 reached, flag = {}'.format(response.meta['flag']))
+                self.logger.info('2018 reached, flag = 1')
                 next_page = response.xpath("//div/a[contains(@href,'time') and contains(text(),'2018')]/@href").extract()
                 self.logger.info('next_page = {}'.format(next_page[0]))
                 new_page = response.urljoin(next_page[0])
@@ -144,6 +156,7 @@ class FacebookSpider(scrapy.Spider):
         
     def parse_reactions(self,response):
         new = ItemLoader(item=FbcrawlItem(),response=response, parent=response.meta['item'])
+        new.context['lang'] = self.lang           
         new.add_xpath('likes',"//a[contains(@href,'reaction_type=1')]/span/text()")
         new.add_xpath('ahah',"//a[contains(@href,'reaction_type=4')]/span/text()")
         new.add_xpath('love',"//a[contains(@href,'reaction_type=2')]/span/text()")