1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111

package main

import (
    "fmt"
    "log"
    "os"
    "path"
    "runtime/debug"
    "syscall"
    "time"
)

// reclamation begins when free space dips below the lowWaterMark and continues
// until free space exceeds the highWaterMark.
const lowWaterMark = 5_000_000_000 // 5 GB
const highWaterMark = 10_000_000_000 // 10 GB

const reclamationInterval = 1 * time.Second

func (c *cache) reclaim(archiveURL string) (err error) {
    ar := c.archivesByURL[archiveURL]
    if ar == nil {
        err = fmt.Errorf("couldn't find archive for URL %s", archiveURL)
        return
    }
    // remove the archive from the reclamation list and the archivesByURL map.
    c.mutex.Lock()
    delete(c.archivesByURL, archiveURL)
    for i, v := range c.archiveURLsToReclaim {
        if v == archiveURL {
            if i == 0 {
                c.archiveURLsToReclaim = c.archiveURLsToReclaim[1:]
            } else {
                c.archiveURLsToReclaim = append(c.archiveURLsToReclaim[:i], c.archiveURLsToReclaim[i+1:]...)
            }
            break
        }
    }
    c.mutex.Unlock()

    ar.mutex.Lock()
    ar.transitionToState(archiveStateFailed)
    ar.failureReason = fmt.Errorf("this archive is being reclaimed")
    ar.mutex.Unlock()

    log.Printf("reclaiming archive %s", ar.path)
    c.reclaimFiles(ar.path)

    return
}

func (c *cache) reclaimLoop() {
    for {
        err := func () (err error) {
            defer func () {
                if r := recover(); r != nil {
                    err = fmt.Errorf("panic during reclaimIfNeeded: %v\n%v", r, string(debug.Stack()))
                }
            }()
            var bytes uint64
            if bytes, err = availableBytes(c.rootPath); err != nil {
                return
            }
            if bytes > lowWaterMark {
                return
            }
            for bytes < highWaterMark {
                if len(c.archiveURLsToReclaim) == 0 {
                    err = fmt.Errorf("still low on space, even after reclaiming all archives")
                    return
                }
                if err = c.reclaim(c.archiveURLsToReclaim[0]); err != nil {
                    return
                }
                if bytes, err = availableBytes(c.rootPath); err != nil {
                    return
                }
            }
            return
        }()
        if err != nil {
            log.Print(err)
        }
        time.Sleep(reclamationInterval)
    }
}

func availableBytes(path string) (uint64, error) {
    var s syscall.Statfs_t
    if err := syscall.Statfs(path, &s); err != nil {
        return 0, err
    }
    return uint64(s.Bsize) * s.Bavail, nil
}

func (c *cache) reclaimFiles(archivePath string) {
    os.Remove(c.archiveMetadataPath(archivePath))
    reclaimDirectory(path.Join(c.rootPath, archivePath))
    reclaimDirectory(path.Join(c.textPath, archivePath))
}

func reclaimDirectory(directory string) {
    os.RemoveAll(directory)
    // reclaim empty parent directories.
    for {
        directory = path.Dir(directory)
        if err := os.Remove(directory); err != nil {
            break
        }
    }
}