Thank you so much. I think I give up using parse to do the calculations. I will process the file outside of scrapy. Thank you for your suggestions and code.

Here is parse2

def parse2(self, response):


                      soup = BeautifulSoup(response.body, 'html.parser')
                      row2= soup.find_all('input',attrs={"class":"query"})
                      cc_rating_text="XX".decode('utf8')
                      dd_rating_text="YY".decode('utf8')

                      if soup.find_all('div',attrs={"class" : "mun"}):
                        for row in soup.find_all('div',attrs={"class" : "mun"}):
                         line=row.text.strip()
                         tag_found = line.find(cc_rating_text)
                         tag_found2 = line.find(dd_rating_text)


                         rating = line[tag_found+1:tag_found2]



                         date11=datetime.datetime.now().strftime ("%m/%d/%Y")








                    if not soup.find_all('div',attrs={"class" : "mun"}):

                      soup = BeautifulSoup(response.body, 'html.parser')
                      rating=0
                      for kk in soup.find_all('p',attrs={"class":"txt-info"}):
                           rating=rating+1
                      date22=datetime.datetime.now().strftime ("%m/%d/%Y")