You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
 
 
 
 
 
 

100 lines
2.2 KiB

package search
import (
"fmt"
"log"
"net/http"
"strings"
"sync"
"github.com/PuerkitoBio/goquery"
)
func (bing *Bing) Search() []Entity {
bing.Req.url = bing.urlWrap()
log.Printf("req.url: %s\n", bing.Req.url)
resp := &Resp{}
resp, _ = bing.send()
bing.resp = *resp
return bing.toEntityList()
}
func (bing *Bing) urlWrap() (url string) {
return fmt.Sprintf(BingSearch, bing.Req.Q)
}
func (bing *Bing) toEntityList() []Entity {
var resList []Entity
if bing.resp.doc != nil {
var wg sync.WaitGroup
results := make(chan Entity, SearchNum)
done := make(chan bool)
bing.resp.doc.Find("ol#b_results>li[class=b_algo]").EachWithBreak(func(i int, s *goquery.Selection) bool {
if len(resList) >= SearchNum { // 如果已经处理了5篇,停止遍历
return false
}
// For each item found, get the Title
title := s.Find("div[class=b_title]>h2>a").Text()
url := s.Find("div[class=b_title]>h2>a").AttrOr("href", "")
subTitle := s.Find("div[class=b_caption]>p").Text()
entity := Entity{From: BingFrom}
entity.Title = title
entity.SubTitle = subTitle
entity.Url = url
host := strings.ReplaceAll(url, "http://", "")
host = strings.ReplaceAll(host, "https://", "")
entity.Host = strings.Split(host, "/")[0]
wg.Add(1)
go func(entity Entity) {
defer wg.Done()
// 获取详细页面的纯文本内容
content, err := FetchContent(entity.Url)
if err != nil {
log.Printf("Error fetching content for URL %s: %v\n", entity.Url, err)
results <- entity
return
}
entity.Content = content
results <- entity
}(entity)
return true
})
go func() {
wg.Wait()
close(results)
done <- true
}()
for entity := range results {
resList = append(resList, entity)
}
<-done
}
return resList
}
func (bing *Bing) send() (resp *Resp, err error) {
client := &http.Client{
Transport: tr,
}
//提交请求
request, err := http.NewRequest("GET", bing.urlWrap(), nil)
if err != nil {
log.Println(err)
}
//增加header选项
request.Header.Add("User-Agent", UserAgent)
request.Header.Add("Host", BingDomain)
request.Header.Add("Cookie", BingCoolkie)
request.Header.Add("Accept", BingAccept)
return SendDo(client, request)
}