41 lines
1.8 KiB
Python
41 lines
1.8 KiB
Python
import logging
|
||
from utils.WebCrawler import WebCrawler
|
||
|
||
logging.basicConfig(level=logging.INFO)
|
||
|
||
if __name__ == '__main__':
|
||
# 保存地址，分别保存html源文件、处理后text文件
|
||
html_dir = "data/html/tmp_csdn_122513786_html.jsonl"
|
||
text_dir = "data/text/tmp_csdn_122513786_text.jsonl"
|
||
# 下载网页数据
|
||
# https://www.langchain.asia/
|
||
# https://blog.csdn.net/weixin_43791511/article/details/122513786
|
||
# https://zhuanlan.zhihu.com/p/645400277
|
||
# https://www.aliyun.com/?utm_content=se_1014243503
|
||
# 'https://cloud.tencent.com/developer/article/1004500?from=15425'
|
||
base_url = 'https://www.langchain.asia/'
|
||
# 爬取方式：
|
||
## requests和selenium两种方式；requests为简单请求静态网址html内容，js动态数据无法获取；
|
||
## selenium为模拟人行为请求，可获取全部html数据，但请求时间较长10-20s单网页，尽量设置5s以上的time_sleep。
|
||
reptile_lib = "requests"
|
||
method = "get" # 目前只支持get请求
|
||
time_sleep = 4 # 每两次请求间隔时间s
|
||
wc = WebCrawler()
|
||
# 爬取base_url单网址
|
||
wc.webcrawler_single(html_dir=html_dir,
|
||
text_dir=text_dir,
|
||
base_url=base_url,
|
||
reptile_lib=reptile_lib,
|
||
method=method,
|
||
time_sleep=time_sleep
|
||
)
|
||
|
||
# # 爬取base_url页面所有网址，限制target_url_prefix为前缀，默认target_url_prefix=base_url
|
||
# wc.webcrawler_1_degree(html_dir=html_dir,
|
||
# text_dir=text_dir,
|
||
# base_url=base_url,
|
||
# reptile_lib=reptile_lib,
|
||
# method=method,
|
||
# time_sleep=time_sleep
|
||
# )
|