mirror of https://gitee.com/godoos/godoos.git
You can not select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
201 lines
11 KiB
201 lines
11 KiB
package search
|
|
|
|
import (
|
|
"encoding/json"
|
|
"godo/libs"
|
|
"io"
|
|
"log"
|
|
"net/http"
|
|
"net/url"
|
|
"sync"
|
|
|
|
"github.com/PuerkitoBio/goquery"
|
|
"jaytaylor.com/html2text"
|
|
)
|
|
|
|
const (
|
|
SearchNum = 5
|
|
Debug = false
|
|
UserAgent = "user-agent: Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/107.0.0.0 Safari/537.36"
|
|
ProxyURL = "http://127.0.0.1:7890"
|
|
ProxyOpen = false
|
|
|
|
// =================BAIDU==================
|
|
//BaiduCookie = ""
|
|
BaiduCookie = "BIDUPSID=3535F2B8A915447A4839A7DD194BA7B3; PSTM=1617718622; __yjs_duid=1_76a86308fbc1b29f8b2c852e8c9e24fd1620886035244; BD_UPN=123253; MCITY=-%3A; BDUSS=m5TRVBaQjk0NXF0SFpmMHZtN2ZWSTlyZVo2d0ZJTVhlU1BMTUpqbXRrZEl0WFZqSUFBQUFBJCQAAAAAAAAAAAEAAAC8QhUx1MbUqtauwfpoaQAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAEgoTmNIKE5jc; BDUSS_BFESS=m5TRVBaQjk0NXF0SFpmMHZtN2ZWSTlyZVo2d0ZJTVhlU1BMTUpqbXRrZEl0WFZqSUFBQUFBJCQAAAAAAAAAAAEAAAC8QhUx1MbUqtauwfpoaQAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAEgoTmNIKE5jc; BDSFRCVID=rY_OJeC62lCrte6jotU8bVRNE2SdnBRTH6aotxm4whxuChbLecyMEG0Pyf8g0KubiKd_ogKK0eOTHktF_2uxOjjg8UtVJeC6EG0Ptf8g0f5; H_BDCLCKID_SF=JJkO_D_atKvDqTrP-trf5DCShUFsyMvlB2Q-XPoO3KJnMU74M4Qtj4Fp3-ON--QiW5cpoMbgylRp8P3y0bb2DUA1y4vp5MnqQeTxoUJ2fnRJEUcGqj5Ah--ebPRiJPQ9QgbWLpQ7tt5W8ncFbT7l5hKpbt-q0x-jLTnhVn0MBCK0MC09j6KhDTPVKgTa54cbb4o2WbCQQq3O8pcN2b5oQT81jnnHatoh32jZ-pb4bb5vOPQKDpOUWfAkXpJvQnJjt2JxaqRCKhv-Sl5jDh3Me-AsLn6te6jzaIvy0hvctn5cShnc5MjrDRLbXU6BK5vPbNcZ0l8K3l02V-bIe-t2XjQhDNKtt5_jJJIsBP_8aJ7bHn7gbJK_-P4DeP6wexRZ5mAqoDQ6tbbnHRcH3noJ-lKw3MTA-55lba6naIQqa-cKDR38Kp58hPT00tb8Qnb43bRT2PPy5KJvfJo1BTrYhP-UyN3-Wh3725nlMKoaMp78jR093JO4y4Ldj4oxJpOJ5JbMonLafDD3fb7kbP6Eq4D_MfOtetJyaR3OoM7vWJ5WqR7jD5DbMP4qQfcrQxTn3Nrfb-OIJPTKShbXKxonKlLObbOGLM6EfNcZ0l8K3l02V-bRDDcfQJQDQt7JKPRMW20j0l7mWnvMsxA45J7cM4IseboJLfT-0bc4KKJxbnLWeIJEjjCKejbQDGADq6nfb5kXLn7J-J5HfbTkbITjhPrMKRrdWMT-0bFHWpO-bnK-jR7m3j7D3lDNQH5gKP6GLHn7_JjObPnVM-3d2boTM-DTbfb8WxQxtNR80DnjtpvhHRTobhnobUPUDMo9LUvWbgcdot5yBbc8eIna5hjkbfJBQttjQn3hfIkj0DKLK-oj-D8lej7P; BAIDUID=9A9CC424161261B757A36C59CFED76E5:FG=1; BDORZ=B490B5EBF6F3CD402E515D22BCDA1598; H_PS_PSSID=36542_37557_37519_36884_37627_36786_37539_37500_26350_37343_37461; BAIDUID_BFESS=9A9CC424161261B757A36C59CFED76E5:FG=1; BDSFRCVID_BFESS=rY_OJeC62lCrte6jotU8bVRNE2SdnBRTH6aotxm4whxuChbLecyMEG0Pyf8g0KubiKd_ogKK0eOTHktF_2uxOjjg8UtVJeC6EG0Ptf8g0f5; H_BDCLCKID_SF_BFESS=JJkO_D_atKvDqTrP-trf5DCShUFsyMvlB2Q-XPoO3KJnMU74M4Qtj4Fp3-ON--QiW5cpoMbgylRp8P3y0bb2DUA1y4vp5MnqQeTxoUJ2fnRJEUcGqj5Ah--ebPRiJPQ9QgbWLpQ7tt5W8ncFbT7l5hKpbt-q0x-jLTnhVn0MBCK0MC09j6KhDTPVKgTa54cbb4o2WbCQQq3O8pcN2b5oQT81jnnHatoh32jZ-pb4bb5vOPQKDpOUWfAkXpJvQnJjt2JxaqRCKhv-Sl5jDh3Me-AsLn6te6jzaIvy0hvctn5cShnc5MjrDRLbXU6BK5vPbNcZ0l8K3l02V-bIe-t2XjQhDNKtt5_jJJIsBP_8aJ7bHn7gbJK_-P4DeP6wexRZ5mAqoDQ6tbbnHRcH3noJ-lKw3MTA-55lba6naIQqa-cKDR38Kp58hPT00tb8Qnb43bRT2PPy5KJvfJo1BTrYhP-UyN3-Wh3725nlMKoaMp78jR093JO4y4Ldj4oxJpOJ5JbMonLafDD3fb7kbP6Eq4D_MfOtetJyaR3OoM7vWJ5WqR7jD5DbMP4qQfcrQxTn3Nrfb-OIJPTKShbXKxonKlLObbOGLM6EfNcZ0l8K3l02V-bRDDcfQJQDQt7JKPRMW20j0l7mWnvMsxA45J7cM4IseboJLfT-0bc4KKJxbnLWeIJEjjCKejbQDGADq6nfb5kXLn7J-J5HfbTkbITjhPrMKRrdWMT-0bFHWpO-bnK-jR7m3j7D3lDNQH5gKP6GLHn7_JjObPnVM-3d2boTM-DTbfb8WxQxtNR80DnjtpvhHRTobhnobUPUDMo9LUvWbgcdot5yBbc8eIna5hjkbfJBQttjQn3hfIkj0DKLK-oj-D8lej7P; delPer=0; BD_CK_SAM=1; PSINO=3; BA_HECTOR=8581ala48k0la181aga50fm81hm8prs1e; ZFY=MW5tgLZfxnXk8HLzWL5vKMTfHTBgdsBmZudM7p1:BE:Aw:C; sugstore=1; H_PS_645EC=6985pYm8uvzjRIBPLtrwALaCau4%2Fxj7uefaYB7sGfR5qYLEGfRZ8Eyb%2F1oc"
|
|
BaiduUrl = "https://www.baidu.com"
|
|
BaiduDomain = "www.baidu.com"
|
|
BaiduAccept = "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.9"
|
|
BaiduSearch = BaiduUrl + "/s?wd=%s" + "&usm=3&rsv_idx=2&rsv_page=1"
|
|
BaiduFrom = "百度"
|
|
|
|
// =================BING==================
|
|
BingCoolkie = ""
|
|
//BingCoolkie = "MUID=16AE9B2FE9DC6568044F8B3EE8F2640C; MUIDB=16AE9B2FE9DC6568044F8B3EE8F2640C; SRCHD=AF=BDVEHC; SRCHUID=V=2&GUID=7321ED22410B41459C503CB6D2628196&dmnchg=1; _UR=QS=0&TQS=0; imgv=flts=20220704; _tarLang=default=zh-Hans; _TTSS_IN=hist=WyJlbiIsImF1dG8tZGV0ZWN0Il0=; _TTSS_OUT=hist=WyJ6aC1IYW5zIl0=; _HPVN=CS=eyJQbiI6eyJDbiI6MjQsIlN0IjoyLCJRcyI6MCwiUHJvZCI6IlAifSwiU2MiOnsiQ24iOjI0LCJTdCI6MCwiUXMiOjAsIlByb2QiOiJIIn0sIlF6Ijp7IkNuIjoyNCwiU3QiOjEsIlFzIjowLCJQcm9kIjoiVCJ9LCJBcCI6dHJ1ZSwiTXV0ZSI6dHJ1ZSwiTGFkIjoiMjAyMi0wOS0wNlQwMDowMDowMFoiLCJJb3RkIjowLCJHd2IiOjAsIkRmdCI6bnVsbCwiTXZzIjowLCJGbHQiOjAsIkltcCI6Nzd9; ANIMIA=FRE=1; MMCASM=ID=3FEB6F3855CC49E584F2DA61F6E5E44C; ZHCHATSTRONGATTRACT=TRUE; _SS=SID=11649FAAD94F6D9716388DF6D8296CB4&PC=U316; SRCHS=PC=U316; ABDEF=V=13&ABDV=13&MRB=0&MRNB=1668441959593; SUID=M; _EDGE_S=SID=11649FAAD94F6D9716388DF6D8296CB4&ui=zh-cn; SRCHUSR=DOB=20210406&T=1668472757000&TPC=1668472758000; ZHCHATWEAKATTRACT=TRUE; ipv6=hit=1668476360196&t=4; ZHLASTACTIVECHAT=0; ZHSEARCHCHATSTATUS=STATUS=0; SNRHOP=I=&TS=; RECSEARCH=SQs=[{\"q\":\"giac%20%E4%B8%8A%E6%B5%B7\"%2C\"c\":1%2C\"ad\":true}%2C{\"q\":\"sessioncachesize\"%2C\"c\":1%2C\"ad\":false}%2C{\"q\":\"rset\"%2C\"c\":1%2C\"ad\":false}%2C{\"q\":\"xx%3Ainitiatingheapoccupancypercent\"%2C\"c\":1%2C\"ad\":false}%2C{\"q\":\"%E7%AE%A1%E7%90%86%E7%9A%84%E5%B8%B8%E8%AF%86\"%2C\"c\":1%2C\"ad\":true}%2C{\"q\":\"%E7%AE%A1%E7%90%86%E7%9A%84%E5%B8%B8%E8%AF%86%20%E5%BE%B7%E9%B2%81%E5%85%8B\"%2C\"c\":1%2C\"ad\":true}%2C{\"q\":\"%E7%AE%A1%E7%90%86\"%2C\"c\":1%2C\"ad\":true}%2C{\"q\":\"goquery%20%E5%BE%AA%E7%8E%AF\"%2C\"c\":1%2C\"ad\":false}%2C{\"q\":\"yuanbiguo\"%2C\"c\":1%2C\"ad\":false}]; SRCHHPGUSR=SRCHLANGV2=zh-Hans&BRW=W&BRH=S&CW=1396&CH=435&DPR=2&UTC=480&DM=0&WTS=63804069557&HV=1668474094&BZA=0&SRCHLANG=zh-Hans&SW=1440&SH=900&PV=11.2.3&EXLTT=6&SCW=1381&SCH=1408&PRVCW=1396&PRVCH=764"
|
|
BingUrl = "https://cn.bing.com"
|
|
BingDomain = "cn.bing.com"
|
|
BingAccept = "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.9"
|
|
BingSearch = BingUrl + "/search?q=%s" + "&PC=U316&FORM=CHROMN"
|
|
BingFrom = "Bing"
|
|
|
|
// =================Google==================
|
|
//GoogleCookie = ""
|
|
GoogleCookie = "CONSENT=YES+srp.gws-20220523-0-RC1.zh-CN+FX; HSID=AIDFOIfZRXMjhRznJ; SSID=ATYvcUlmXr3mPFerl; APISID=fTLh3HcYiZa0Ch2l/AiukySr7MDg_GhSRo; SAPISID=-JpKVkIJZXDgucyp/AjPiMHGTZYbtqrQbt; __Secure-1PAPISID=-JpKVkIJZXDgucyp/AjPiMHGTZYbtqrQbt; __Secure-3PAPISID=-JpKVkIJZXDgucyp/AjPiMHGTZYbtqrQbt; SEARCH_SAMESITE=CgQIk5YB; SID=Qggrdf-HuR3Im3foqYJWVZDPEay8b5U0-O_E6L489Ppxqy2bY358yp5YaaIs5NjEgyW23g.; __Secure-1PSID=Qggrdf-HuR3Im3foqYJWVZDPEay8b5U0-O_E6L489Ppxqy2brNnvCnxg8UCPQbHqZClMmA.; __Secure-3PSID=Qggrdf-HuR3Im3foqYJWVZDPEay8b5U0-O_E6L489Ppxqy2bJ0s1kXYhOOAgur0ax0qp6w.; OTZ=6777442_24_24__24_; AEC=AakniGNW8IBpZgW_-IGOh45Otu8fCK-Ty1n0eSZhgs05euV7s2hhmWLhCQ; NID=511=k96f3xX7KMB1Wvhg478KURvWSAd53Y3rTkVG3bMb1FhtdJwQUbUiHJOnVTgFgW0Mad_1X5gKnWLMt0lPHl33nQdVCTzEiitFOC2dIicYusLP1zl_L6Wh9l-XO6x9VRh4ZxGlcu1bCmlEbBYvz2eL2ioCJ3RMGZtGcr6_1tdpGZH8DRcj3c8X6FxRJqcX5peACa9pGELYmZk4TfOYiUON7p4ht5MTkfA4-hmLUD_JQOT_bK4Aub5SzhXukCFBt5qx2UMkKKtheWlK6cRlR-EIqbC0ppccy2pUdT0YyfE; DV=c77sl5f1j0hXACm1QWzNqJ3oPLjLSRge9TXKlwa5UAEAAMBLUjyomhJKegAAABD63jYjJ8URJwAAAKJBqsVeeJWmDgAAAA; 1P_JAR=2022-11-22-00; SIDCC=AIKkIs0Jk8Fs8wt-yHi9yEM8vFzc6y1bmzVm7mYb3eELJ0t_5yixpCQToLaIDwlOnk3yTvFwKw; __Secure-1PSIDCC=AIKkIs2Sk9zwJs1YfCaHpcFU4ZAKHzDO7Atrzk3b-uXf384Xuc1nQ0U41MiX_VQcEBrJ0TSR1ZY; __Secure-3PSIDCC=AIKkIs1iF4en_Ln8Jonhw8Q79QB_dwbKwcw0khXXIX0vsIwOjBgxmO8dd6kfi10evURBz7YdDg"
|
|
GoogleUrl = "https://www.google.com"
|
|
GoogleDomain = "www.google.com"
|
|
GoogleAccept = "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.9"
|
|
GoogleSearch = GoogleUrl + "/search?q=%s" + "&ie=UTF-8"
|
|
GoogleFrom = "Google"
|
|
|
|
// =================Wx==================
|
|
WxCookie = ""
|
|
//WxCookie = "SUID=F84DCC781539960A000000006235D4E8; SUV=1647695080857391; ssuid=6792259580; weixinIndexVisited=1; IPLOC=CN3100; ABTEST=0|1668775030|v1; JSESSIONID=aaa2PBWFjHNS8hQ8_tfpy; cd=1668913493&0f942166ea05ede01cfe88195d36508d; rd=tyllllllll20WBOSYTuBqQ2iuqV0WBOqAfJbLZllll9llllxVllll5@@@@@@@@@@; ld=6Zllllllll20WBOSYTuBqQ2DdNH0WBOqAfJbLZllll9lllllVklll5@@@@@@@@@@; LSTMV=217%2C66; LCLKINT=1482; PHPSESSID=udut3pen5cml0b9849o68jch40; SNUID=9D60E0542D28C24D1776D6A42DF58CE7; ariaDefaultTheme=undefined"
|
|
WxUrl = "https://weixin.sogou.com"
|
|
WxDomain = "weixin.sogou.com"
|
|
WxAccept = "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.9"
|
|
WxSearch = WxUrl + "/weixin?type=2&s_from=input&query=%s&ie=utf8&_sug_=n&_sug_type_="
|
|
WxFrom = "微信公众号"
|
|
)
|
|
|
|
type SearchEngine interface {
|
|
Search() (result []Entity)
|
|
urlWrap() (url string)
|
|
toEntityList() (entityList []Entity)
|
|
send() (resp *Resp, err error)
|
|
}
|
|
|
|
type Baidu struct {
|
|
Req Req
|
|
resp Resp
|
|
}
|
|
|
|
type Bing struct {
|
|
Req Req
|
|
resp Resp
|
|
}
|
|
|
|
type Req struct {
|
|
Q string
|
|
url string
|
|
userAgent string
|
|
http.Cookie
|
|
}
|
|
|
|
type Resp struct {
|
|
code int
|
|
body string
|
|
doc *goquery.Document
|
|
}
|
|
|
|
type Entity struct {
|
|
Title string `json:"title"`
|
|
Host string `json:"host"`
|
|
Url string `json:"url"`
|
|
SubTitle string `json:"subTitle"`
|
|
From string `json:"from"`
|
|
Content string `json:"content"`
|
|
}
|
|
|
|
func SearchWebhandler(w http.ResponseWriter, r *http.Request) {
|
|
//log.Println(r.URL)
|
|
_ = r.ParseForm()
|
|
q := r.Form.Get("q")
|
|
if q == "" {
|
|
libs.ErrorMsg(w, "查询内容不能为空")
|
|
return
|
|
}
|
|
q = url.QueryEscape(q)
|
|
//log.Printf("查询内容:%s\n", q)
|
|
jsonResult := SearchWeb(q)
|
|
|
|
// 构造返回
|
|
body, err := json.Marshal(jsonResult)
|
|
if err != nil {
|
|
libs.ErrorMsg(w, "解析错误")
|
|
return
|
|
}
|
|
libs.SuccessMsg(w, body, "success")
|
|
}
|
|
func SearchWeb(q string) []Entity {
|
|
|
|
q = url.QueryEscape(q)
|
|
//log.Printf("查询内容:%s\n", q)
|
|
jsonResult := []Entity{}
|
|
|
|
array := [...]SearchEngine{
|
|
&Bing{Req: Req{Q: q}},
|
|
&Baidu{Req: Req{Q: q}},
|
|
}
|
|
|
|
var wg sync.WaitGroup
|
|
c := make(chan []Entity, len(array))
|
|
|
|
for _, engine := range array {
|
|
wg.Add(1)
|
|
go func(engine SearchEngine) {
|
|
defer wg.Done()
|
|
result := engine.Search()
|
|
c <- result
|
|
}(engine)
|
|
}
|
|
|
|
go func() {
|
|
wg.Wait()
|
|
close(c)
|
|
}()
|
|
|
|
for result := range c {
|
|
if result != nil {
|
|
jsonResult = append(jsonResult, result...)
|
|
}
|
|
}
|
|
|
|
return jsonResult
|
|
}
|
|
func SendDo(client *http.Client, request *http.Request) (*Resp, error) {
|
|
resp := &Resp{code: 200}
|
|
|
|
//处理返回结果
|
|
response, e := client.Do(request)
|
|
if response == nil {
|
|
resp.code = 500
|
|
log.Printf("response nil: %v\n", e)
|
|
return resp, nil
|
|
}
|
|
if response.StatusCode != 200 {
|
|
resp.code = response.StatusCode
|
|
log.Printf("status code error: %d %s\n", response.StatusCode, response.Status)
|
|
return resp, nil
|
|
}
|
|
defer response.Body.Close()
|
|
// Load the HTML document
|
|
doc, err := goquery.NewDocumentFromReader(response.Body)
|
|
if err != nil {
|
|
log.Println(err)
|
|
}
|
|
|
|
resp.code = 200
|
|
resp.doc = doc
|
|
|
|
return resp, nil
|
|
}
|
|
func FetchContent(url string) (content string, err error) {
|
|
resp, err := http.Get(url)
|
|
if err != nil {
|
|
return "", err
|
|
}
|
|
defer resp.Body.Close()
|
|
|
|
body, errRead := io.ReadAll(resp.Body)
|
|
if errRead != nil {
|
|
return "", err
|
|
}
|
|
text, err := html2text.FromString(string(body), html2text.Options{PrettyTables: false})
|
|
if err != nil {
|
|
return "", err
|
|
}
|
|
// if len(text) > 1000 {
|
|
// text = text[:1000]
|
|
// }
|
|
return text, nil
|
|
}
|
|
|