1 f10bc8eb 2024-02-07 aa // Copyright (c) 2024 Alexander Arkhipov <aa@manpager.org>
3 f10bc8eb 2024-02-07 aa // Permission to use, copy, modify, and distribute this software for any
4 f10bc8eb 2024-02-07 aa // purpose with or without fee is hereby granted, provided that the above
5 f10bc8eb 2024-02-07 aa // copyright notice and this permission notice appear in all copies.
7 f10bc8eb 2024-02-07 aa // THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
8 f10bc8eb 2024-02-07 aa // WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
9 f10bc8eb 2024-02-07 aa // MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
10 f10bc8eb 2024-02-07 aa // ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
11 f10bc8eb 2024-02-07 aa // WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
12 f10bc8eb 2024-02-07 aa // ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
13 f10bc8eb 2024-02-07 aa // OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
29 f10bc8eb 2024-02-07 aa var qflag = flag.Bool("q", false, "be quiet")
30 f10bc8eb 2024-02-07 aa var pflag = flag.Int("p", 1, "number of parallel downloads")
32 f10bc8eb 2024-02-07 aa type filename struct {
33 f10bc8eb 2024-02-07 aa n int // how many times the url has been accessed
34 f10bc8eb 2024-02-07 aa name string // end-file name
35 f10bc8eb 2024-02-07 aa tmpfiles []string // temporary file names
38 f10bc8eb 2024-02-07 aa // filemap maps URLs to corresponding filenames
39 f10bc8eb 2024-02-07 aa var filemap = make(map[string]filename)
41 f10bc8eb 2024-02-07 aa func getUrl(url, f string, ch chan int) {
42 f10bc8eb 2024-02-07 aa defer func() { ch <- 0 }()
44 f10bc8eb 2024-02-07 aa rm := func() {
49 f10bc8eb 2024-02-07 aa fmt.Println("GET", url)
52 f10bc8eb 2024-02-07 aa fp, err := os.Create(f)
53 f10bc8eb 2024-02-07 aa if err != nil {
54 f10bc8eb 2024-02-07 aa fmt.Fprintln(os.Stderr, err)
58 f10bc8eb 2024-02-07 aa defer fp.Close()
59 f10bc8eb 2024-02-07 aa fmt.Println("created", fp.Name())
61 f10bc8eb 2024-02-07 aa resp, err := http.Get(url)
62 f10bc8eb 2024-02-07 aa if err != nil {
63 f10bc8eb 2024-02-07 aa fmt.Fprintln(os.Stderr, err)
68 f10bc8eb 2024-02-07 aa buf := make([]byte, 4096)
69 f10bc8eb 2024-02-07 aa reader := bufio.NewReader(resp.Body)
70 f10bc8eb 2024-02-07 aa writer := bufio.NewWriter(fp)
72 f10bc8eb 2024-02-07 aa for readErr := error(nil); readErr == nil; {
73 f10bc8eb 2024-02-07 aa n, readErr := io.ReadFull(reader, buf)
74 f10bc8eb 2024-02-07 aa if readErr == io.EOF {
77 f10bc8eb 2024-02-07 aa if readErr != nil && readErr != io.ErrUnexpectedEOF {
78 f10bc8eb 2024-02-07 aa fmt.Fprintln(os.Stderr, readErr)
83 f10bc8eb 2024-02-07 aa _, writeErr := writer.Write(buf[:n])
84 f10bc8eb 2024-02-07 aa if writeErr != nil {
85 f10bc8eb 2024-02-07 aa fmt.Fprintln(os.Stderr, writeErr)
90 f10bc8eb 2024-02-07 aa writer.Flush()
93 f10bc8eb 2024-02-07 aa func prepUrl(url, d string) (string, error) {
94 f10bc8eb 2024-02-07 aa if !(strings.HasPrefix(url, "http://") || strings.HasPrefix(url, "https://")) {
95 f10bc8eb 2024-02-07 aa url = "http://" + url
98 f10bc8eb 2024-02-07 aa fmentry := filemap[url]
99 f10bc8eb 2024-02-07 aa defer func() { filemap[url] = fmentry }()
101 f10bc8eb 2024-02-07 aa var fname string
