1 f10bc8eb 2024-02-07 aa // Copyright (c) 2024 Alexander Arkhipov <aa@manpager.org>
3 f10bc8eb 2024-02-07 aa // Permission to use, copy, modify, and distribute this software for any
4 f10bc8eb 2024-02-07 aa // purpose with or without fee is hereby granted, provided that the above
5 f10bc8eb 2024-02-07 aa // copyright notice and this permission notice appear in all copies.
7 f10bc8eb 2024-02-07 aa // THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
8 f10bc8eb 2024-02-07 aa // WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
9 f10bc8eb 2024-02-07 aa // MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
10 f10bc8eb 2024-02-07 aa // ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
11 f10bc8eb 2024-02-07 aa // WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
12 f10bc8eb 2024-02-07 aa // ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
13 f10bc8eb 2024-02-07 aa // OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
29 f10bc8eb 2024-02-07 aa var qflag = flag.Bool("q", false, "be quiet")
30 f10bc8eb 2024-02-07 aa var pflag = flag.Int("p", 1, "number of parallel downloads")
32 f10bc8eb 2024-02-07 aa type filename struct {
33 f10bc8eb 2024-02-07 aa n int // how many times the url has been accessed
34 f10bc8eb 2024-02-07 aa name string // end-file name
35 f10bc8eb 2024-02-07 aa tmpfiles []string // temporary file names
38 f10bc8eb 2024-02-07 aa // filemap maps URLs to corresponding filenames
39 f10bc8eb 2024-02-07 aa var filemap = make(map[string]filename)
41 f10bc8eb 2024-02-07 aa func getUrl(url, f string, ch chan int) {
42 f10bc8eb 2024-02-07 aa defer func() { ch <- 0 }()
44 f10bc8eb 2024-02-07 aa rm := func() {
49 f10bc8eb 2024-02-07 aa fmt.Println("GET", url)
52 f10bc8eb 2024-02-07 aa fp, err := os.Create(f)
53 f10bc8eb 2024-02-07 aa if err != nil {
54 f10bc8eb 2024-02-07 aa fmt.Fprintln(os.Stderr, err)
58 f10bc8eb 2024-02-07 aa defer fp.Close()
59 f10bc8eb 2024-02-07 aa fmt.Println("created", fp.Name())
61 f10bc8eb 2024-02-07 aa resp, err := http.Get(url)
62 f10bc8eb 2024-02-07 aa if err != nil {
63 f10bc8eb 2024-02-07 aa fmt.Fprintln(os.Stderr, err)
68 f10bc8eb 2024-02-07 aa buf := make([]byte, 4096)
69 f10bc8eb 2024-02-07 aa reader := bufio.NewReader(resp.Body)
70 f10bc8eb 2024-02-07 aa writer := bufio.NewWriter(fp)
72 f10bc8eb 2024-02-07 aa for readErr := error(nil); readErr == nil; {
73 f10bc8eb 2024-02-07 aa n, readErr := io.ReadFull(reader, buf)
74 f10bc8eb 2024-02-07 aa if readErr == io.EOF {
77 f10bc8eb 2024-02-07 aa if readErr != nil && readErr != io.ErrUnexpectedEOF {
78 f10bc8eb 2024-02-07 aa fmt.Fprintln(os.Stderr, readErr)
83 f10bc8eb 2024-02-07 aa _, writeErr := writer.Write(buf[:n])
84 f10bc8eb 2024-02-07 aa if writeErr != nil {
85 f10bc8eb 2024-02-07 aa fmt.Fprintln(os.Stderr, writeErr)
90 f10bc8eb 2024-02-07 aa writer.Flush()
93 f10bc8eb 2024-02-07 aa func prepUrl(url, d string) (string, error) {
94 f10bc8eb 2024-02-07 aa if !(strings.HasPrefix(url, "http://") || strings.HasPrefix(url, "https://")) {
95 f10bc8eb 2024-02-07 aa url = "http://" + url
98 f10bc8eb 2024-02-07 aa fmentry := filemap[url]
99 f10bc8eb 2024-02-07 aa defer func() { filemap[url] = fmentry }()
101 f10bc8eb 2024-02-07 aa var fname string
103 f10bc8eb 2024-02-07 aa _, fname, _ = strings.Cut(url, "://")
104 f10bc8eb 2024-02-07 aa _, fname, _ = strings.Cut(fname, "/")
105 f10bc8eb 2024-02-07 aa parts := strings.Split(fname, "/")
106 f10bc8eb 2024-02-07 aa fname = parts[len(parts)-1]
107 f10bc8eb 2024-02-07 aa if fname == "" {
108 f10bc8eb 2024-02-07 aa fname = "index.html"
111 f10bc8eb 2024-02-07 aa tmpfp, err := os.CreateTemp(d, fname+"*")
112 f10bc8eb 2024-02-07 aa if err != nil {
113 f10bc8eb 2024-02-07 aa return "", err
115 f10bc8eb 2024-02-07 aa defer tmpfp.Close()
117 f10bc8eb 2024-02-07 aa fmentry.name = fname
118 f10bc8eb 2024-02-07 aa fmentry.tmpfiles = append(fmentry.tmpfiles, tmpfp.Name())
120 f10bc8eb 2024-02-07 aa return url, nil
123 f10bc8eb 2024-02-07 aa func main() {
126 f10bc8eb 2024-02-07 aa if *pflag < 1 {
127 f10bc8eb 2024-02-07 aa fmt.Fprintln(os.Stderr, "can't do less than 1 parallel downloads")
131 f10bc8eb 2024-02-07 aa var urls []string
133 f10bc8eb 2024-02-07 aa tmpdir, err := os.MkdirTemp(".", ".goget*")
134 f10bc8eb 2024-02-07 aa if err != nil {
135 f10bc8eb 2024-02-07 aa fmt.Fprintln(os.Stderr, err)
138 f10bc8eb 2024-02-07 aa defer func() {
139 f10bc8eb 2024-02-07 aa rename := func(url string) {
140 f10bc8eb 2024-02-07 aa fentry := filemap[url]
141 f10bc8eb 2024-02-07 aa defer func() {
142 f10bc8eb 2024-02-07 aa fentry.tmpfiles = fentry.tmpfiles[1:]
143 f10bc8eb 2024-02-07 aa filemap[url] = fentry
145 f10bc8eb 2024-02-07 aa os.Rename(fentry.tmpfiles[0], fentry.name)
146 f10bc8eb 2024-02-07 aa // Ignoring ErrNotExist since the temporary file might
147 f10bc8eb 2024-02-07 aa // have been removed on purpose.
149 f10bc8eb 2024-02-07 aa // TODO It would be better to have such purposeful
150 f10bc8eb 2024-02-07 aa // removals marked explicitly.
151 f10bc8eb 2024-02-07 aa if err != nil && !errors.Is(err, fs.ErrNotExist) {
152 f10bc8eb 2024-02-07 aa fmt.Fprintln(os.Stderr, err)
156 f10bc8eb 2024-02-07 aa for _, url := range urls {
160 f10bc8eb 2024-02-07 aa err := os.Remove(tmpdir)
161 f10bc8eb 2024-02-07 aa if err != nil {
162 f10bc8eb 2024-02-07 aa fmt.Fprintln(os.Stderr, err)
167 f10bc8eb 2024-02-07 aa for _, arg := range flag.Args() {
168 f10bc8eb 2024-02-07 aa url, err := prepUrl(arg, tmpdir)
169 f10bc8eb 2024-02-07 aa if err != nil {
170 f10bc8eb 2024-02-07 aa fmt.Fprintln(os.Stderr, err)
173 f10bc8eb 2024-02-07 aa urls = append(urls, url)
176 f10bc8eb 2024-02-07 aa ch := make(chan int, *pflag)
177 f10bc8eb 2024-02-07 aa routines := 0
178 f10bc8eb 2024-02-07 aa for _, url := range urls {
179 f10bc8eb 2024-02-07 aa if routines >= *pflag {
183 f10bc8eb 2024-02-07 aa if fmentry, ok := filemap[url]; ok {
184 f10bc8eb 2024-02-07 aa go getUrl(url, fmentry.tmpfiles[fmentry.n], ch)
186 f10bc8eb 2024-02-07 aa filemap[url] = fmentry
191 f10bc8eb 2024-02-07 aa for routines > 0 {
