一个简单的go语言爬虫

佚名 6年前 (2019-04-14) 随笔 1845人围观抢沙发百度已收录

package main

import (
    "bufio"
    "fmt"
    "golang.org/x/net/html/charset"
    "golang.org/x/text/encoding"
    "golang.org/x/text/transform"
    "io"
    "io/ioutil"
    "net/http"
)

func determineEncoding(r io.Reader) encoding.Encoding {
       //编码判断
    bytes, err := bufio.NewReader(r).Peek(1024)
    if err != nil {
        panic(err)
    }
    e, _, _ := charset.DetermineEncoding(bytes, ",")
    return e

}
func main() {
    resp, err := http.Get("http://www.baidu.com")
    if err != nil {
        panic(err)
    }
    defer resp.Body.Close() //最后关闭，先进后出

    if resp.StatusCode != http.StatusOK {
        fmt.Println("Error:status code", resp.StatusCode)
        return
    }
    e := determineEncoding(resp.Body)
    utf8Reader := transform.NewReader(resp.Body, e.NewDecoder())
    all, err := ioutil.ReadAll(utf8Reader)
    //utf8Reader:=transform.NewReader(resp.Body,simplifiedchinese.GBK.NewDecoder())

    //下面是不加编码判断的
    //all, err := ioutil.ReadAll(resp.Body)
    if err != nil {
        panic(err)
    }
    fmt.Printf("%s\n", all)

}