Automate the Boring Stuff with Go, Ch. 11: Web Requests

// fetch-page makes one HTTP GET the right way: a client with a timeout, a context, a
// User-Agent that says who you are, a status check, and a body that is always closed.
//
//	go run ./ch11/projects/fetch-page https://www.t0ul.com/
package main

import (
	"context"
	"errors"
	"fmt"
	"io"
	"net/http"
	"os"
	"time"
)

const userAgent = "automate-the-boring-stuff-with-go/2026 (+https://www.t0ul.com)"

// Client is the one HTTP client every project in this chapter shares the shape of.
var Client = &http.Client{Timeout: 15 * time.Second}

// Fetch GETs url and returns the body, up to limit bytes, plus the content type.
func Fetch(ctx context.Context, pageURL string, limit int64) (body []byte, contentType string, err error) {
	req, err := http.NewRequestWithContext(ctx, http.MethodGet, pageURL, nil)
	if err != nil {
		return nil, "", err
	}
	req.Header.Set("User-Agent", userAgent)
	resp, err := Client.Do(req)
	if err != nil {
		return nil, "", fmt.Errorf("get %s: %w", pageURL, err)
	}
	defer resp.Body.Close()
	if resp.StatusCode != http.StatusOK {
		return nil, "", fmt.Errorf("get %s: %s", pageURL, resp.Status)
	}
	body, err = io.ReadAll(io.LimitReader(resp.Body, limit))
	if err != nil {
		return nil, "", fmt.Errorf("read %s: %w", pageURL, err)
	}
	return body, resp.Header.Get("Content-Type"), nil
}

func run(args []string) error {
	if len(args) != 1 {
		return errors.New("usage: fetch-page <url>")
	}
	ctx, cancel := context.WithTimeout(context.Background(), 20*time.Second)
	defer cancel()
	body, ct, err := Fetch(ctx, args[0], 1<<20)
	if err != nil {
		return err
	}
	fmt.Printf("content-type: %s\nbytes: %d\n\n", ct, len(body))
	if len(body) > 500 {
		body = body[:500]
	}
	fmt.Printf("%s\n", body)
	return nil
}

func main() {
	if err := run(os.Args[1:]); err != nil {
		fmt.Fprintln(os.Stderr, "error:", err)
		os.Exit(1)
	}
}

// search-term builds a URL with query parameters from whatever you type, calls a JSON
// API and prints the results. The API is Open Library's book search, which needs no key.
//
//	go run ./ch11/projects/search-term go programming language
package main

import (
	"context"
	"encoding/json"
	"errors"
	"fmt"
	"net/http"
	"net/url"
	"os"
	"strings"
	"time"
)

var client = &http.Client{Timeout: 15 * time.Second}

// Result is the part of one Open Library hit we care about.
type Result struct {
	Title       string   `json:"title"`
	Authors     []string `json:"author_name"`
	PublishYear int      `json:"first_publish_year"`
}

// SearchURL builds <base>?q=<terms>&limit=<n>, escaping the terms properly.
func SearchURL(base, terms string, limit int) string {
	q := url.Values{}
	q.Set("q", terms)
	q.Set("limit", fmt.Sprint(limit))
	q.Set("fields", "title,author_name,first_publish_year")
	return base + "?" + q.Encode()
}

// Search calls the API and decodes its "docs" array.
func Search(ctx context.Context, searchURL string) ([]Result, error) {
	req, err := http.NewRequestWithContext(ctx, http.MethodGet, searchURL, nil)
	if err != nil {
		return nil, err
	}
	req.Header.Set("User-Agent", "automate-the-boring-stuff-with-go/2026")
	resp, err := client.Do(req)
	if err != nil {
		return nil, fmt.Errorf("search: %w", err)
	}
	defer resp.Body.Close()
	if resp.StatusCode != http.StatusOK {
		return nil, fmt.Errorf("search: %s", resp.Status)
	}
	var payload struct {
		Docs []Result `json:"docs"`
	}
	if err := json.NewDecoder(resp.Body).Decode(&payload); err != nil {
		return nil, fmt.Errorf("decode: %w", err)
	}
	return payload.Docs, nil
}

func run(args []string) error {
	if len(args) == 0 {
		return errors.New("usage: search-term <words to search for>")
	}
	terms := strings.Join(args, " ")
	u := SearchURL("https://openlibrary.org/search.json", terms, 5)
	fmt.Println("GET", u)
	ctx, cancel := context.WithTimeout(context.Background(), 20*time.Second)
	defer cancel()
	results, err := Search(ctx, u)
	if err != nil {
		return err
	}
	for i, r := range results {
		fmt.Printf("%d. %s (%d) by %s\n", i+1, r.Title, r.PublishYear, strings.Join(r.Authors, ", "))
	}
	return nil
}

func main() {
	if err := run(os.Args[1:]); err != nil {
		fmt.Fprintln(os.Stderr, "error:", err)
		os.Exit(1)
	}
}

// save-page downloads a URL to a file without ever holding the whole thing in memory:
// the response body streams straight into the file with io.Copy.
//
//	go run ./ch11/projects/save-page -out go-spec.html https://go.dev/ref/spec
package main

import (
	"context"
	"errors"
	"flag"
	"fmt"
	"io"
	"net/http"
	"net/url"
	"os"
	"path"
	"time"
)

var client = &http.Client{Timeout: 2 * time.Minute}

// Download streams url into path and returns the number of bytes written.
func Download(ctx context.Context, rawURL, dest string) (int64, error) {
	req, err := http.NewRequestWithContext(ctx, http.MethodGet, rawURL, nil)
	if err != nil {
		return 0, err
	}
	req.Header.Set("User-Agent", "automate-the-boring-stuff-with-go/2026")
	resp, err := client.Do(req)
	if err != nil {
		return 0, fmt.Errorf("get: %w", err)
	}
	defer resp.Body.Close()
	if resp.StatusCode != http.StatusOK {
		return 0, fmt.Errorf("get %s: %s", rawURL, resp.Status)
	}
	f, err := os.Create(dest)
	if err != nil {
		return 0, fmt.Errorf("create %s: %w", dest, err)
	}
	n, copyErr := io.Copy(f, resp.Body)
	if err := f.Close(); copyErr == nil && err != nil {
		copyErr = err
	}
	if copyErr != nil {
		return n, fmt.Errorf("save %s: %w", dest, copyErr)
	}
	return n, nil
}

// DefaultName picks a file name from the last part of the URL path, or index.html.
func DefaultName(rawURL string) string {
	u, err := url.Parse(rawURL)
	if err != nil {
		return "download"
	}
	base := path.Base(u.Path)
	if base == "." || base == "/" || base == "" {
		return "index.html"
	}
	return base
}

func run(args []string) error {
	flags := flag.NewFlagSet("save-page", flag.ContinueOnError)
	out := flags.String("out", "", "file to write (default: last part of the URL)")
	if err := flags.Parse(args); err != nil {
		return err
	}
	if flags.NArg() != 1 {
		return errors.New("usage: save-page [-out file] <url>")
	}
	rawURL := flags.Arg(0)
	if *out == "" {
		*out = DefaultName(rawURL)
	}
	n, err := Download(context.Background(), rawURL, *out)
	if err != nil {
		return err
	}
	fmt.Printf("saved %d bytes to %s\n", n, *out)
	return nil
}

func main() {
	if err := run(os.Args[1:]); err != nil {
		fmt.Fprintln(os.Stderr, "error:", err)
		os.Exit(1)
	}
}

// download-xkcd saves the most recent xkcd comics to a folder. xkcd publishes a JSON
// description of every comic, so there is no HTML to scrape: ask for the metadata, then
// fetch the image it points to. Comics already on disk are skipped, and there is a short
// pause between requests because it is someone else's server.
//
//	go run ./ch11/projects/download-xkcd -n 5 -dir xkcd
package main

import (
	"context"
	"encoding/json"
	"errors"
	"flag"
	"fmt"
	"io"
	"net/http"
	"os"
	"path"
	"path/filepath"
	"time"
)

var client = &http.Client{Timeout: 30 * time.Second}

// Comic is the subset of xkcd's JSON we use.
type Comic struct {
	Num   int    `json:"num"`
	Title string `json:"title"`
	Img   string `json:"img"`
	Alt   string `json:"alt"`
}

// Fetcher knows where the API lives, so tests can point it at a local server.
type Fetcher struct {
	Base  string        // "https://xkcd.com"
	Pause time.Duration // between requests
}

// Latest returns the newest comic's metadata.
func (f Fetcher) Latest(ctx context.Context) (Comic, error) {
	return f.getJSON(ctx, f.Base+"/info.0.json")
}

// Get returns comic number n.
func (f Fetcher) Get(ctx context.Context, n int) (Comic, error) {
	return f.getJSON(ctx, fmt.Sprintf("%s/%d/info.0.json", f.Base, n))
}

func (f Fetcher) getJSON(ctx context.Context, apiURL string) (Comic, error) {
	var c Comic
	req, err := http.NewRequestWithContext(ctx, http.MethodGet, apiURL, nil)
	if err != nil {
		return c, err
	}
	req.Header.Set("User-Agent", "automate-the-boring-stuff-with-go/2026")
	resp, err := client.Do(req)
	if err != nil {
		return c, fmt.Errorf("get %s: %w", apiURL, err)
	}
	defer resp.Body.Close()
	if resp.StatusCode != http.StatusOK {
		return c, fmt.Errorf("get %s: %s", apiURL, resp.Status)
	}
	if err := json.NewDecoder(resp.Body).Decode(&c); err != nil {
		return c, fmt.Errorf("decode %s: %w", apiURL, err)
	}
	return c, nil
}

// SaveImage downloads c's image into dir as <num>-<original name> and reports whether it
// downloaded anything (false when the file already existed).
func (f Fetcher) SaveImage(ctx context.Context, c Comic, dir string) (bool, error) {
	name := filepath.Join(dir, fmt.Sprintf("%d-%s", c.Num, path.Base(c.Img)))
	if _, err := os.Stat(name); err == nil {
		return false, nil
	}
	req, err := http.NewRequestWithContext(ctx, http.MethodGet, c.Img, nil)
	if err != nil {
		return false, err
	}
	resp, err := client.Do(req)
	if err != nil {
		return false, fmt.Errorf("image %d: %w", c.Num, err)
	}
	defer resp.Body.Close()
	if resp.StatusCode != http.StatusOK {
		return false, fmt.Errorf("image %d: %s", c.Num, resp.Status)
	}
	tmp := name + ".part"
	out, err := os.Create(tmp)
	if err != nil {
		return false, err
	}
	if _, err := io.Copy(out, resp.Body); err != nil {
		out.Close()
		return false, fmt.Errorf("write %s: %w", tmp, err)
	}
	if err := out.Close(); err != nil {
		return false, err
	}
	return true, os.Rename(tmp, name)
}

// DownloadRecent saves the n newest comics and returns how many files it wrote.
func (f Fetcher) DownloadRecent(ctx context.Context, n int, dir string) (int, error) {
	if err := os.MkdirAll(dir, 0o755); err != nil {
		return 0, err
	}
	latest, err := f.Latest(ctx)
	if err != nil {
		return 0, err
	}
	written := 0
	for num := latest.Num; num > latest.Num-n && num > 0; num-- {
		c := latest
		if num != latest.Num {
			time.Sleep(f.Pause)
			if c, err = f.Get(ctx, num); err != nil {
				return written, err
			}
		}
		saved, err := f.SaveImage(ctx, c, dir)
		if err != nil {
			return written, err
		}
		if saved {
			written++
			fmt.Printf("#%d %q\n", c.Num, c.Title)
		} else {
			fmt.Printf("#%d already here\n", c.Num)
		}
	}
	return written, nil
}

func run(args []string) error {
	flags := flag.NewFlagSet("download-xkcd", flag.ContinueOnError)
	n := flags.Int("n", 5, "how many recent comics")
	dir := flags.String("dir", "xkcd", "folder to save into")
	if err := flags.Parse(args); err != nil {
		return err
	}
	if *n < 1 {
		return errors.New("-n must be at least 1")
	}
	f := Fetcher{Base: "https://xkcd.com", Pause: 500 * time.Millisecond}
	ctx, cancel := context.WithTimeout(context.Background(), 5*time.Minute)
	defer cancel()
	written, err := f.DownloadRecent(ctx, *n, *dir)
	if err != nil {
		return err
	}
	fmt.Printf("%d new file(s) in %s\n", written, *dir)
	return nil
}

func main() {
	if err := run(os.Args[1:]); err != nil {
		fmt.Fprintln(os.Stderr, "error:", err)
		os.Exit(1)
	}
}

// scrape-books reads a catalogue page and prints title, price and star rating for every
// book on it. The default target is books.toscrape.com, a site that exists to be
// scraped. Scraping a site that did not invite it is a different conversation: read its
// robots.txt and terms, go slowly, identify yourself, and prefer an API when there is one.
//
//	go run ./ch11/projects/scrape-books
//	go run ./ch11/projects/scrape-books -url https://books.toscrape.com/catalogue/page-2.html
package main

import (
	"context"
	"flag"
	"fmt"
	"io"
	"net/http"
	"os"
	"strings"
	"time"

	"github.com/PuerkitoBio/goquery"
)

var client = &http.Client{Timeout: 20 * time.Second}

// Book is one row of the catalogue.
type Book struct {
	Title  string
	Price  string
	Rating string
}

// Parse extracts the books from an HTML page in the books.toscrape.com layout.
func Parse(r io.Reader) ([]Book, error) {
	doc, err := goquery.NewDocumentFromReader(r)
	if err != nil {
		return nil, fmt.Errorf("parse html: %w", err)
	}
	var books []Book
	doc.Find("article.product_pod").Each(func(_ int, s *goquery.Selection) {
		title, _ := s.Find("h3 a").Attr("title")
		rating := ""
		if classes, ok := s.Find("p.star-rating").Attr("class"); ok {
			rating = strings.TrimSpace(strings.TrimPrefix(classes, "star-rating"))
		}
		books = append(books, Book{
			Title:  title,
			Price:  strings.TrimSpace(s.Find("p.price_color").Text()),
			Rating: rating,
		})
	})
	return books, nil
}

// Scrape fetches url and parses it.
func Scrape(ctx context.Context, pageURL string) ([]Book, error) {
	req, err := http.NewRequestWithContext(ctx, http.MethodGet, pageURL, nil)
	if err != nil {
		return nil, err
	}
	req.Header.Set("User-Agent", "automate-the-boring-stuff-with-go/2026 (+https://www.t0ul.com)")
	resp, err := client.Do(req)
	if err != nil {
		return nil, fmt.Errorf("get %s: %w", pageURL, err)
	}
	defer resp.Body.Close()
	if resp.StatusCode != http.StatusOK {
		return nil, fmt.Errorf("get %s: %s", pageURL, resp.Status)
	}
	return Parse(resp.Body)
}

func run(args []string) error {
	flags := flag.NewFlagSet("scrape-books", flag.ContinueOnError)
	pageURL := flags.String("url", "https://books.toscrape.com/", "catalogue page to read")
	if err := flags.Parse(args); err != nil {
		return err
	}
	ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
	defer cancel()
	books, err := Scrape(ctx, *pageURL)
	if err != nil {
		return err
	}
	for _, b := range books {
		fmt.Printf("%-8s %-6s %s\n", b.Price, b.Rating, b.Title)
	}
	fmt.Fprintf(os.Stderr, "%d book(s) on %s\n", len(books), *pageURL)
	return nil
}

func main() {
	if err := run(os.Args[1:]); err != nil {
		fmt.Fprintln(os.Stderr, "error:", err)
		os.Exit(1)
	}
}

// browser-chromedp drives a real, headless Chrome: load a page, wait for an element,
// read the title and a heading, click a link, and save a screenshot. Use it when a page
// only makes sense after JavaScript has run; for plain HTML, net/http is enough.
//
//	go run ./ch11/projects/browser-chromedp https://books.toscrape.com/
package main

import (
	"context"
	"errors"
	"fmt"
	"os"
	"time"

	"github.com/chromedp/chromedp"
)

// allocatorOptions is chromedp's headless default set, plus --no-sandbox when running
// on a CI runner (GitHub sets CI=true). Chrome's sandbox needs unprivileged user
// namespaces, which Ubuntu 24.04 runners restrict, so Chrome refuses to start without the
// flag there. Keep the sandbox on your own machine: it is what stands between a hostile
// page and the rest of your system.
func allocatorOptions() []chromedp.ExecAllocatorOption {
	opts := append([]chromedp.ExecAllocatorOption{}, chromedp.DefaultExecAllocatorOptions[:]...)
	if os.Getenv("CI") != "" {
		opts = append(opts, chromedp.NoSandbox)
	}
	return opts
}

// Visit loads url, returns the page title and the text of the first h1, and writes a
// screenshot to shot. Pass an empty shot to skip the screenshot.
func Visit(ctx context.Context, pageURL, shot string) (title, heading string, err error) {
	allocCtx, cancel := chromedp.NewExecAllocator(ctx, allocatorOptions()...)
	defer cancel()
	tab, cancel := chromedp.NewContext(allocCtx)
	defer cancel()

	if err := chromedp.Do(tab, chromedp.Navigate(pageURL), chromedp.WaitVisible("h1")); err != nil {
		return "", "", fmt.Errorf("load %s: %w", pageURL, err)
	}
	if title, err = chromedp.Run(tab, chromedp.Title()); err != nil {
		return "", "", err
	}
	if heading, err = chromedp.Run(tab, chromedp.Text("h1")); err != nil {
		return "", "", err
	}
	if shot != "" {
		png, err := chromedp.Run(tab, chromedp.FullScreenshot(90))
		if err != nil {
			return title, heading, fmt.Errorf("screenshot: %w", err)
		}
		if err := os.WriteFile(shot, png, 0o644); err != nil {
			return title, heading, err
		}
	}
	return title, heading, nil
}

func run(args []string) error {
	if len(args) != 1 {
		return errors.New("usage: browser-chromedp <url>")
	}
	ctx, cancel := context.WithTimeout(context.Background(), 60*time.Second)
	defer cancel()
	title, heading, err := Visit(ctx, args[0], "screenshot.png")
	if err != nil {
		return err
	}
	fmt.Printf("title:   %s\nheading: %s\nsaved screenshot.png\n", title, heading)
	return nil
}

func main() {
	if err := run(os.Args[1:]); err != nil {
		fmt.Fprintln(os.Stderr, "error:", err)
		os.Exit(1)
	}
}