import requests
from bs4 import BeautifulSoup   #html解析器模块
import re   #正则表达式模块
visited_links = set()  # 用集合存储已经访问过的标题
i = 0
file = open( 1.txt , w ,encoding= utf8 )  #打开文件设置写入
content1 =   
while True:
  i += 1
  url1 = ***** 
  headers = {
     User-Agent :  Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/58.0.3029.110 Safari/537.3 ,
     Referer :   ,
     Cookie :   ,
  }
  try:
    response = requests.get(url1, headers=headers,timeout=3)
    response.encoding =  utf-8   # 设置编码
    soup = BeautifulSoup(response.text,  html.parser )  # 给内容设置解析器
    prettify = soup.prettify()  # 格式化内容
    title = soup.title.string  # 获取标题
    pattern = r ^
http://www.crazyant.net/page/\d+$ 
    if re.match(pattern,url1) and title !=  未找到页面 – 学Python :  # 没跳出这个页面则爬取
      entry_title = soup.find_all(class_= entry-title )
      for entry_title_1 in entry_title:
        a_tag =  entry_title_1.find( a )  #寻找a
        a_title = a_tag.string  #获取标题
        if a_tag:
          href = a_tag.get( href )  #获取链接
          if href not in visited_links:  # 判断集合里面有没有这个标题
            visited_links.add(href)  # 集合里面没有则添加标题
            content1 += f {href}\t{a_title}\n 
            #file.write(content1)  #写入内容
            print(content1)
    else:
      print( 没找到页面 )
      file.close()
      break
  except:
    print( 错误 )
    file.close()
    exit()