|--原 QueryList 采集代码实例
QueryList 采集代码实例主要看QueryList的部分，其他的不用理会 20200328//采集traileraddict的首页
public static function collectInfoTraileraddictHome(){
$s=time();
$filename=config('website.logs.collect');//记录采集的日志文件
$baseUrl='';
//待采集的目标页面，PHPHub教程区
$page = 'https://www.traileraddict.com/';//traileraddict的首页
//列表选择器 bxslider
$rang = '#homemenu >li';
//采集规则
$rules = array(
//文章标题
'title' => ['a','title'],
//文章链接
'url' => ['a','href'],
//图片
'source_image' => ['img','src']
);
//采集
$data = \QL\QueryList::Query($page,$rules,$rang)->data;
$rang2= '#top_features >ul >li';
$rules2 = [
//文章标题
'title' => ['h2','text'],
//文章链接
'url' => ['a','href'],
//图片
'source_image' => ['','style','',function($content) use($baseUrl){
$content=str_replace('background-image:url(', '', $content);
$content=str_replace(')', '', $content);
$content='https:'.$content;
return $content;}],
];
$data2 = \QL\QueryList::Query($page,$rules2,$rang2)->data;
$rang3= '.featured_box';
$rules3 = [
//文章标题
'title' => ['a','text'],
//文章链接
'url' => ['a','href'],
//图片
'source_image' => ['a','href','',function($content) {
return '';
}],
];
$data3 = \QL\QueryList::Query($page,$rules3,$rang3)->data;
$i=0;//插入多少条到数据库计数
$j=0;//采集过的二级链接计数
$datas=array_merge($data,$data2,$data3);
$num=count($datas);
浏览更多内容请先登录。
立即注册
更新于：2022-09-01 23:17:15
|--原 Curl类采集，之前网上找的
之前网上找的别人写的专门用来采集的类，封装了php的curl 20200328 class cURL { var $headers; var $user_agent; v...之前网上找的别人写的专门用来采集的类，封装了php的curl 20200328 class cURL { var $headers; var $user_agent; var $compression; var $cookie_file; var $proxy; /** * 初始化 * * @param string $cookies * @param string $cookie * @param string $compression * @param string $proxy */ function cURL($cookies = TRUE, $cookie = 'cookies.txt', $compression = 'gzip', $proxy = '') { $this->headers [] = 'Accept: image/gif, image/x-bitmap, image/jpeg, image/pjpeg'; $this->headers [] = 'Connection: Keep-Alive'; $this->headers [] = 'Content-type: application/x-www-form-urlencoded;charset=UTF-8'; $this->user_agent = 'Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 5.1; .NET CLR 1.0.3705; .NET CLR 1.1.4322; Media Center PC 4.0)'; $this->compression = $compression; $this->proxy = $proxy; $this->cookies = $cookies; if ($this->cookies == TRUE) $this->cookie ( $cookie ); } /** * 配置cookie * * @param unknown $cookie_file */ function cookie($cookie_file) { if (file_exists ( $cookie_file )) { $this->cookie_file = $cookie_file; } else { fopen ( $cookie_file, 'w' ) or $this->error ( 'The cookie file could not be opened. Make sure this directory has the correct permissions' ); $this->cookie_file = $cookie_file; fclose ( $this->cookie_file ); } } /** * get方式打开页面 * * @param unknown $url * @return mixed */ function get($url) { $process = curl_init ( $url ); curl_setopt ( $process, CURLOPT_HTTPHEADER, $this->headers ); curl_setopt ( $process, CURLOPT_HEADER, 0 ); curl_setopt ( $process, CURLOPT_USERAGENT, $this->user_agent ); if ($this->cookies == TRUE) c
浏览更多内容请先登录。
立即注册
更新于：2020-03-28 12:50:12
|--原 自己用curl函数封装的一些采集函数
自己封装的一些采集函数，这里记录下 20200328 /** * () 13N2y19 1203 * @access public * @param $search_str...自己封装的一些采集函数，这里记录下 20200328 /** * () 13N2y19 1203 * @access public * @param $search_str 获取要搜索的字符串 * @return $contents 获取的网页内容 */ function curl_google($search_str){ $cookie_file = "google.txt"; $str_urlencode=urlencode($search_str); $url = "http://www.google.com/search?q={$str_urlencode}"; $ch = curl_init(); curl_setopt($ch, CURLOPT_URL, $url); curl_setopt($ch, CURLOPT_USERAGENT, $_SERVER['HTTP_USER_AGENT']); curl_setopt($ch, CURLOPT_RETURNTRANSFER, 1); curl_setopt($ch, CURLOPT_FOLLOWLOCATION, 1); curl_setopt($ch, CURLOPT_COOKIEJAR, $cookie_file); $contents = curl_exec($ch); curl_close($ch); return $contents; } function curl_baidu($search_str,$pn){ if(isset($pn)===false){ $pn=0; } $cookie_file = "baidu.txt"; $search_str=iconv('utf-8','gbk',$search_str); $str_urlencode=urlencode($search_str); $url="http://www.baidu.com/s?wd={$str_urlencode}&pn={$pn}"; $ch = curl_init(); curl_setopt($ch, CURLOPT_URL, $url); curl_setopt($ch, CURLOPT_USERAGENT, $_SERVER['HTTP_USER_AGENT']); curl_setopt($ch, CURLOPT_RETURNTRANSFER, 1); curl_setopt($ch, CURLOPT_FOLLOWLOCATION, 1); curl_setopt($ch, CURLOPT_COOKIEJAR, $cookie_file); $contents = curl_exec($ch); $contents=iconv('gbk','utf-8',$contents); curl_close($ch); return $contents; } function curl_baiduzhidao($search_str,$pn,$sort){ if(isset($pn)===false){ $pn=0; } if(isset($sort)===false){ $sort=0; } $cookie_file = "baidu.txt"; $search_str=iconv('utf-8','gbk',$search_str); $str_urlencode=urlencode($search_str); $url="http://zhidao.baidu.com/search?word={$search_str}&lm=0&rn=10&sort={$sort}&ie=gbk&pn={$pn}"; $ch = curl_init(); curl_setopt($ch, CURLOPT_URL, $url); curl_setopt($ch, CURLOPT_USERAGENT, $_SERVER['HTTP_USER_AGENT']); curl_setopt($ch, CURLOPT_RETURNTRANSFER, 1); curl_setopt($ch, CURLOPT_FOLLOWLOCATION, 1); curl_setopt($ch, CURLOPT_COOKIEJAR, $cookie_file); $contents = curl_exec($ch); $contents=iconv('gbk','utf-8',$contents); curl_close($ch); return $contents; } function curl_mtimeid_moviedetails($mtime_id){ $cookie_file = "mtime.txt"; $url="http://movie.mtime.com/{$mtime_id}/details.html#menu"; $ch = curl_init(); curl_setopt($ch, CURLOPT_URL, $url); curl_setopt($ch, CURLOPT_USERAGENT, $_SERVER['HTTP_USER_AGENT']); curl_setopt($ch, CURLOPT_RETURNTRANSFER, 1); curl_setopt($ch, CURLOPT_FOLLOWLOCATION, 1); curl_se
浏览更多内容请先登录。
立即注册
更新于：2020-03-28 12:53:08
相关内容
这里专门开个帖子用来整理采集遇到的问题
Linux中使用curl命令访问https站点4种常见错误和解决方法
使用 curl 进行 ssl 认证 -文章是百度搜curl.cainfo找到的
网上之前找的封装php curl的类，小巧且实用，用了挺久
采集的时候把目标网页的内容输出到页面调试的问题
PHP实现抓取百度搜索结果并分析数据结构
CentOS 8 安装Puppeteer 记录
windows wamp SSL certificate problem: unable to get local issuer cert...
安装Puppeteer插件，PHP采集实现抓取百度搜索结果并分析数据结构
采集时遇到报错，去github.com查资料，遇到Github网站打不开的问题，网上找的...
PHP采集时报错Failed to launch the browser process puppeteer
Win7安装nodejs，之后在sublime运行，之后再安装 puppeteer采集网页
puppeteer爬取豆瓣电影信息
解决centos运行node项目puppeteer时chrome错误问题
How to Setup Puppeteer In CentOS 7 用spatie/browsershot成功采集百度...
cnpm 安装的扩展的路径 不好找，觉得还是用npm安装，用国内的镜像源
nodejs 报错 Error: EPERM: operation not permitted, mkdir‘xxxxxxxxx...
新的chrome headless模式 headless=new
nodejs 报错 Error: Could not find Chrome (ver. 119.0.6045.105). This ...
Error: Could not find Chrome 运行js脚本直接执行ok，用php的exec执行脚...
[PHP] 网盘搜索引擎-采集爬取百度网盘分享文件实现网盘搜索
WEB技术
WEB技术之前端技术
WEB技术之后端技术
WEB应用转手机APP，手机APP制作平台推荐
WEB应用与手机APP
Android相关
2017 年 Web 开发工程师技术发展路线图
session:手动删除客户端上的所有cookie,再次访问的时候为什么还是登录状态?
推荐内容
