2 Commits
Author SHA1 Message Date
a.gritsenkoandClaude Opus 5.5 4a03d58193 CI: release every push to main with the next patch tag
build / build (push) Successful in 1m3s
Pushes to main get the next vX.Y.(Z+1) tag, are built with that version and
published as a Gitea release with the deb/rpm and commit subjects as notes.
Manual v* tags still release that exact version; already attached assets are
skipped so a tag-triggered rerun does not duplicate them.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-10-03 10:29:51 +03:00
a.gritsenkoandClaude Opus 5.5 0cdee29359 ZFS: dedup toggle per pool and dedup option for new datasets
build / build (push) Successful in 1m4s
Pool cards show the dedup ratio and can set dedup=on|off on the pool root
dataset (inherited by children); the new-dataset form gets a dedup option.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-10-03 09:30:38 +03:00
10 changed files with 257 additions and 27 deletions
+48 -12
View File
@@ -1,5 +1,8 @@
name: build name: build
# Every push to main is released: it gets the next patch tag (v0.1.3 -> v0.1.4)
# and a Gitea release with the deb/rpm. Pushing a v* tag by hand releases that
# exact version instead (use it to bump minor/major).
on: on:
push: push:
branches: ["**"] branches: ["**"]
@@ -14,6 +17,8 @@ jobs:
runs-on: ubuntu-latest runs-on: ubuntu-latest
steps: steps:
- uses: https://gitea.com/actions/checkout@v4 - uses: https://gitea.com/actions/checkout@v4
with:
fetch-depth: 0 # tags + history for the version and release notes
- uses: https://gitea.com/actions/setup-go@v5 - uses: https://gitea.com/actions/setup-go@v5
with: with:
@@ -24,6 +29,27 @@ jobs:
go vet ./... go vet ./...
go test ./... go test ./...
- name: release version
id: ver
if: github.event_name == 'push' && (github.ref == 'refs/heads/main' || startsWith(github.ref, 'refs/tags/v'))
run: |
set -euo pipefail
git fetch --tags --force -q
if [[ "$GITHUB_REF" == refs/tags/* ]]; then
tag="${GITHUB_REF#refs/tags/}"
else
# re-run of an already released commit keeps its tag
tag=$(git tag --points-at HEAD | grep -E '^v[0-9]+\.[0-9]+\.[0-9]+$' | sort -V | tail -1 || true)
if [ -z "$tag" ]; then
last=$(git tag | grep -E '^v[0-9]+\.[0-9]+\.[0-9]+$' | sort -V | tail -1 || true)
IFS=. read -r major minor patch <<<"${last:-v0.0.0}"
tag="${major}.${minor}.$((patch + 1))"
fi
fi
echo "release $tag"
echo "tag=$tag" >> "$GITHUB_OUTPUT"
echo "VERSION=${tag#v}" >> "$GITHUB_ENV"
- name: install nfpm - name: install nfpm
run: go install github.com/goreleaser/nfpm/v2/cmd/nfpm@latest run: go install github.com/goreleaser/nfpm/v2/cmd/nfpm@latest
@@ -42,31 +68,41 @@ jobs:
dist/*.rpm dist/*.rpm
- name: publish release - name: publish release
if: startsWith(github.ref, 'refs/tags/') if: steps.ver.outputs.tag != ''
env: env:
GITEA_TOKEN: ${{ secrets.GITHUB_TOKEN }} GITEA_TOKEN: ${{ secrets.GITHUB_TOKEN }}
API: ${{ github.api_url }}/repos/${{ github.repository }} API: ${{ github.api_url }}/repos/${{ github.repository }}
TAG: ${{ steps.ver.outputs.tag }}
run: | run: |
set -euo pipefail set -euo pipefail
tag="${GITHUB_REF#refs/tags/}" auth="Authorization: token $GITEA_TOKEN"
release_id=$(curl -sf -X POST "$API/releases" \ # release notes: commit subjects since the previous release tag
-H "Authorization: token $GITEA_TOKEN" \ prev=$(git tag --merged HEAD | grep -E '^v[0-9]+\.[0-9]+\.[0-9]+$' | grep -vx "$TAG" | sort -V | tail -1 || true)
-H "Content-Type: application/json" \ git log --no-merges --pretty='- %s' ${prev:+"$prev"..}HEAD > notes.md
-d "{\"tag_name\":\"$tag\",\"name\":\"$tag\"}" \ body=$(python3 -c 'import json,os,sys; print(json.dumps({"tag_name": os.environ["TAG"], "name": os.environ["TAG"], "target_commitish": os.environ["GITHUB_SHA"], "body": open("notes.md").read()}))')
# creating the release also creates the tag on this commit
release_id=$(curl -sf -X POST "$API/releases" -H "$auth" \
-H "Content-Type: application/json" -d "$body" \
| grep -oP '"id":\s*\K[0-9]+' | head -1) || \ | grep -oP '"id":\s*\K[0-9]+' | head -1) || \
release_id=$(curl -sf "$API/releases/tags/$tag" \ release_id=$(curl -sf "$API/releases/tags/$TAG" -H "$auth" \
-H "Authorization: token $GITEA_TOKEN" \
| grep -oP '"id":\s*\K[0-9]+' | head -1) | grep -oP '"id":\s*\K[0-9]+' | head -1)
if [ -z "$release_id" ]; then if [ -z "$release_id" ]; then
echo "could not create or find a release for tag $tag" >&2 echo "could not create or find a release for tag $TAG" >&2
exit 1 exit 1
fi fi
existing=$(curl -sf "$API/releases/$release_id/assets" -H "$auth" || true)
for f in dist/*.deb dist/*.rpm; do for f in dist/*.deb dist/*.rpm; do
[ -e "$f" ] || continue [ -e "$f" ] || continue
curl -sf -X POST "$API/releases/$release_id/assets?name=$(basename "$f")" \ name=$(basename "$f")
-H "Authorization: token $GITEA_TOKEN" \ if grep -q "\"name\":\"$name\"" <<<"$existing"; then
-F "attachment=@${f}" echo "$name already attached"
continue
fi
curl -sf -X POST "$API/releases/$release_id/assets?name=$name" \
-H "$auth" -F "attachment=@${f}"
done done
echo "released $TAG: ${{ github.server_url }}/${{ github.repository }}/releases/tag/$TAG"
+11 -2
View File
@@ -35,6 +35,8 @@ type Pool struct {
Status string `json:"status,omitempty"` Status string `json:"status,omitempty"`
Action string `json:"action,omitempty"` Action string `json:"action,omitempty"`
Vdevs []Vdev `json:"vdevs"` Vdevs []Vdev `json:"vdevs"`
// DedupRatio is the pool-wide dedup ratio (1.00 = nothing deduplicated).
DedupRatio float64 `json:"dedup_ratio"`
} }
type Dataset struct { type Dataset struct {
@@ -43,11 +45,12 @@ type Dataset struct {
Avail uint64 `json:"avail"` Avail uint64 `json:"avail"`
Refer uint64 `json:"refer"` Refer uint64 `json:"refer"`
Mountpoint string `json:"mountpoint"` Mountpoint string `json:"mountpoint"`
Dedup string `json:"dedup"` // dedup property: off, on, verify, sha256, ...
} }
// ReadPools combines zpool list and zpool status for every pool. // ReadPools combines zpool list and zpool status for every pool.
func ReadPools() ([]Pool, error) { func ReadPools() ([]Pool, error) {
out, err := run(20*time.Second, "zpool", "list", "-Hp", "-o", "name,size,alloc,free,cap,health,frag") out, err := run(20*time.Second, "zpool", "list", "-Hp", "-o", "name,size,alloc,free,cap,health,frag,dedup")
if err != nil { if err != nil {
return nil, err return nil, err
} }
@@ -63,7 +66,7 @@ func ReadPools() ([]Pool, error) {
} }
func ReadDatasets() ([]Dataset, error) { func ReadDatasets() ([]Dataset, error) {
out, err := run(20*time.Second, "zfs", "list", "-Hp", "-t", "filesystem", "-o", "name,used,avail,refer,mountpoint") out, err := run(20*time.Second, "zfs", "list", "-Hp", "-t", "filesystem", "-o", "name,used,avail,refer,mountpoint,dedup")
if err != nil { if err != nil {
return nil, err return nil, err
} }
@@ -84,6 +87,9 @@ func parseZpoolList(b []byte) []Pool {
p.Free, _ = strconv.ParseUint(f[3], 10, 64) p.Free, _ = strconv.ParseUint(f[3], 10, 64)
p.CapPct = atoiPct(f[4]) p.CapPct = atoiPct(f[4])
p.FragPct = atoiPct(f[6]) p.FragPct = atoiPct(f[6])
if len(f) > 7 {
p.DedupRatio, _ = strconv.ParseFloat(strings.TrimSuffix(strings.TrimSpace(f[7]), "x"), 64)
}
if p.CapPct == 0 && p.Size > 0 { if p.CapPct == 0 && p.Size > 0 {
p.CapPct = int(p.Alloc * 100 / p.Size) p.CapPct = int(p.Alloc * 100 / p.Size)
} }
@@ -220,6 +226,9 @@ func parseZfsList(b []byte) []Dataset {
d.Used, _ = strconv.ParseUint(f[1], 10, 64) d.Used, _ = strconv.ParseUint(f[1], 10, 64)
d.Avail, _ = strconv.ParseUint(f[2], 10, 64) d.Avail, _ = strconv.ParseUint(f[2], 10, 64)
d.Refer, _ = strconv.ParseUint(f[3], 10, 64) d.Refer, _ = strconv.ParseUint(f[3], 10, 64)
if len(f) > 5 {
d.Dedup = f[5]
}
out = append(out, d) out = append(out, d)
} }
return out return out
+13
View File
@@ -119,6 +119,15 @@ func TestParseZpoolList(t *testing.T) {
if pools[1].CapPct != 90 || pools[1].FragPct != 0 { if pools[1].CapPct != 90 || pools[1].FragPct != 0 {
t.Errorf("pool1 = %+v", pools[1]) t.Errorf("pool1 = %+v", pools[1])
} }
if pools[0].DedupRatio != 0 {
t.Errorf("dedup ratio without the column = %v", pools[0].DedupRatio)
}
// with the dedup column; -p prints "1.35", without -p it is "1.35x"
pools = parseZpoolList([]byte("a\t1\t1\t0\t100\tONLINE\t0\t1.35\nb\t1\t1\t0\t100\tONLINE\t0\t2.00x\n"))
if len(pools) != 2 || pools[0].DedupRatio != 1.35 || pools[1].DedupRatio != 2 {
t.Errorf("dedup ratio = %+v", pools)
}
} }
func TestParseZfsList(t *testing.T) { func TestParseZfsList(t *testing.T) {
@@ -127,4 +136,8 @@ func TestParseZfsList(t *testing.T) {
if len(ds) != 2 || ds[1].Name != "nas_pool_1_8_gb/appdata" || ds[1].Avail != 200 { if len(ds) != 2 || ds[1].Name != "nas_pool_1_8_gb/appdata" || ds[1].Avail != 200 {
t.Errorf("datasets = %+v", ds) t.Errorf("datasets = %+v", ds)
} }
ds = parseZfsList([]byte("tank\t1\t2\t3\t/tank\ton\n"))
if len(ds) != 1 || ds[0].Dedup != "on" || ds[0].Mountpoint != "/tank" {
t.Errorf("dedup column: %+v", ds)
}
} }
+43 -6
View File
@@ -124,7 +124,9 @@ type zfsVM struct {
S sched.Status S sched.Status
Th config.Thresholds Th config.Thresholds
Datasets []datasetRow Datasets []datasetRow
Form createForm // last submitted values (re-rendered after an error) // PoolDedup maps a pool to the dedup property of its root dataset.
PoolDedup map[string]string
Form createForm // last submitted values (re-rendered after an error)
} }
type datasetRow struct { type datasetRow struct {
@@ -134,15 +136,18 @@ type datasetRow struct {
} }
type createForm struct { type createForm struct {
Pool, Name, Owner, Compression, Quota string Pool, Name, Owner, Compression, Dedup, Quota string
} }
func (s *Server) zfsView(st sched.Status, form createForm) zfsVM { func (s *Server) zfsView(st sched.Status, form createForm) zfsVM {
vm := zfsVM{S: st, Th: s.cfg.Get().Thresholds, Form: form} vm := zfsVM{S: st, Th: s.cfg.Get().Thresholds, Form: form, PoolDedup: map[string]string{}}
if vm.Form.Owner == "" { if vm.Form.Owner == "" {
vm.Form.Owner = defaultOwner vm.Form.Owner = defaultOwner
} }
for _, d := range st.Datasets { for _, d := range st.Datasets {
if !strings.Contains(d.Name, "/") {
vm.PoolDedup[d.Name] = d.Dedup
}
row := datasetRow{Dataset: d, DomID: strings.NewReplacer("/", "-", ".", "_", ":", "_").Replace(d.Name)} row := datasetRow{Dataset: d, DomID: strings.NewReplacer("/", "-", ".", "_", ":", "_").Replace(d.Name)}
row.CanDestroy = strings.Contains(d.Name, "/") && d.Refer < zfsadm.EmptyRefer row.CanDestroy = strings.Contains(d.Name, "/") && d.Refer < zfsadm.EmptyRefer
for _, o := range st.Datasets { for _, o := range st.Datasets {
@@ -170,6 +175,7 @@ func (s *Server) zfsCreate(w http.ResponseWriter, r *http.Request) {
Name: strings.Trim(strings.TrimSpace(r.FormValue("name")), "/"), Name: strings.Trim(strings.TrimSpace(r.FormValue("name")), "/"),
Owner: strings.TrimSpace(r.FormValue("owner")), Owner: strings.TrimSpace(r.FormValue("owner")),
Compression: r.FormValue("compression"), Compression: r.FormValue("compression"),
Dedup: r.FormValue("dedup"),
Quota: strings.ToUpper(strings.TrimSpace(r.FormValue("quota"))), Quota: strings.ToUpper(strings.TrimSpace(r.FormValue("quota"))),
} }
st := s.sch.Status() st := s.sch.Status()
@@ -191,20 +197,20 @@ func (s *Server) zfsCreate(w http.ResponseWriter, r *http.Request) {
return return
} }
} }
for _, err := range []error{zfsadm.ValidateOwner(f.Owner), zfsadm.ValidateCompression(f.Compression), zfsadm.ValidateSize(f.Quota)} { for _, err := range []error{zfsadm.ValidateOwner(f.Owner), zfsadm.ValidateCompression(f.Compression), zfsadm.ValidateDedup(f.Dedup), zfsadm.ValidateSize(f.Quota)} {
if err != nil { if err != nil {
fail(err.Error()) fail(err.Error())
return return
} }
} }
err := s.admin.Create(full, zfsadm.CreateOpts{Compression: f.Compression, Quota: f.Quota, Owner: f.Owner}) err := s.admin.Create(full, zfsadm.CreateOpts{Compression: f.Compression, Dedup: f.Dedup, Quota: f.Quota, Owner: f.Owner})
s.sch.RefreshStorage() s.sch.RefreshStorage()
if err != nil { if err != nil {
log.Printf("zfs create %s: %v", full, err) log.Printf("zfs create %s: %v", full, err)
fail(err.Error()) fail(err.Error())
return return
} }
log.Printf("dataset %s created (owner %s, compression %q, quota %q)", full, f.Owner, f.Compression, f.Quota) log.Printf("dataset %s created (owner %s, compression %q, dedup %q, quota %q)", full, f.Owner, f.Compression, f.Dedup, f.Quota)
http.Redirect(w, r, "/zfs?msg="+url.QueryEscape("Dataset "+full+" created")+"#"+url.PathEscape(f.Pool), http.StatusSeeOther) http.Redirect(w, r, "/zfs?msg="+url.QueryEscape("Dataset "+full+" created")+"#"+url.PathEscape(f.Pool), http.StatusSeeOther)
} }
@@ -243,6 +249,37 @@ func (s *Server) zfsDestroy(w http.ResponseWriter, r *http.Request) {
http.Redirect(w, r, "/zfs?msg="+url.QueryEscape("Dataset "+name+" destroyed"), http.StatusSeeOther) http.Redirect(w, r, "/zfs?msg="+url.QueryEscape("Dataset "+name+" destroyed"), http.StatusSeeOther)
} }
// zfsDedup turns dedup on/off on a pool's root dataset (inherited by children).
func (s *Server) zfsDedup(w http.ResponseWriter, r *http.Request) {
if err := r.ParseForm(); err != nil {
http.Error(w, err.Error(), http.StatusBadRequest)
return
}
pool := strings.TrimSpace(r.FormValue("pool"))
value := r.FormValue("dedup")
st := s.sch.Status()
fail := func(msg string) {
s.render(w, r, "zfs", pageData{Title: "ZFS", Data: s.zfsView(st, createForm{}), Alerts: len(st.Alerts), Error: "Dedup not changed: " + msg})
}
if !hasPool(st, pool) {
fail("unknown pool " + pool)
return
}
if value != "on" && value != "off" {
fail("dedup must be on or off")
return
}
err := s.admin.SetDedup(pool, value)
s.sch.RefreshStorage()
if err != nil {
log.Printf("zfs set dedup=%s %s: %v", value, pool, err)
fail(err.Error())
return
}
log.Printf("dedup=%s set on pool %s", value, pool)
http.Redirect(w, r, "/zfs?msg="+url.QueryEscape("Dedup "+value+" for "+pool)+"#"+url.PathEscape(pool), http.StatusSeeOther)
}
func hasPool(st sched.Status, name string) bool { func hasPool(st sched.Status, name string) bool {
for _, p := range st.Pools { for _, p := range st.Pools {
if p.Name == name { if p.Name == name {
+1
View File
@@ -73,6 +73,7 @@ func New(cfg *config.Store, st *store.Store, ring *history.Ring, sch *sched.Sche
s.mux.HandleFunc("GET /zfs", s.zfs) s.mux.HandleFunc("GET /zfs", s.zfs)
s.mux.HandleFunc("POST /zfs/create", s.zfsCreate) s.mux.HandleFunc("POST /zfs/create", s.zfsCreate)
s.mux.HandleFunc("POST /zfs/destroy", s.zfsDestroy) s.mux.HandleFunc("POST /zfs/destroy", s.zfsDestroy)
s.mux.HandleFunc("POST /zfs/dedup", s.zfsDedup)
s.mux.HandleFunc("GET /alerts", s.alerts) s.mux.HandleFunc("GET /alerts", s.alerts)
s.mux.HandleFunc("POST /alerts/{id}/{action}", s.alertAction) s.mux.HandleFunc("POST /alerts/{id}/{action}", s.alertAction)
s.mux.HandleFunc("GET /shares", s.sharesPage) s.mux.HandleFunc("GET /shares", s.sharesPage)
+23
View File
@@ -13,6 +13,7 @@
<div><span class="muted">fragmentation</span> {{.FragPct}}%</div> <div><span class="muted">fragmentation</span> {{.FragPct}}%</div>
<div><span class="muted">last scrub</span> {{if .LastScrub}}{{dtp .LastScrub}} ({{agop .LastScrub}}){{else}}never{{end}}</div> <div><span class="muted">last scrub</span> {{if .LastScrub}}{{dtp .LastScrub}} ({{agop .LastScrub}}){{else}}never{{end}}</div>
<div><span class="muted">errors</span> {{if .Errors}}{{.Errors}}{{else}}—{{end}}</div> <div><span class="muted">errors</span> {{if .Errors}}{{.Errors}}{{else}}—{{end}}</div>
<div><span class="muted">dedup</span> {{with index $.Data.PoolDedup .Name}}{{.}}{{else}}—{{end}}{{if .DedupRatio}} (ratio {{printf "%.2f" .DedupRatio}}x){{end}}</div>
</div> </div>
<div class="bar"><span class="fill {{capClass .CapPct $.Data.Th.PoolCapacityPct 95}}" style="width:{{.CapPct}}%"></span></div> <div class="bar"><span class="fill {{capClass .CapPct $.Data.Th.PoolCapacityPct 95}}" style="width:{{.CapPct}}%"></span></div>
{{if .Scan}}<p class="small"><span class="muted">scan:</span> {{.Scan}}</p>{{end}} {{if .Scan}}<p class="small"><span class="muted">scan:</span> {{.Scan}}</p>{{end}}
@@ -31,6 +32,21 @@
{{end}} {{end}}
</tbody> </tbody>
</table></div> </table></div>
{{$dedup := index $.Data.PoolDedup .Name}}
<details class="dedup dedup-{{.Name}}">
<summary>Deduplication</summary>
<form method="post" action="/zfs/dedup" class="inline">
<input type="hidden" name="pool" value="{{.Name}}">
<label>Dedup for {{.Name}}
<select name="dedup">
<option value="off" {{if or (eq $dedup "off") (eq $dedup "")}}selected{{end}}>off</option>
<option value="on" {{if and (ne $dedup "off") (ne $dedup "")}}selected{{end}}>on</option>
</select>
</label>
<button type="submit">Apply</button>
</form>
<p class="small muted">Sets dedup on the pool root dataset; child datasets inherit it unless they override it. Only newly written data is deduplicated, and turning it off keeps the existing dedup table. The dedup table needs a lot of RAM (up to ~5 GB per 1 TB of unique data) &mdash; enable it only for data with many duplicate blocks.</p>
</details>
<details class="newds newds-{{.Name}}" {{if eq $form.Pool .Name}}open{{end}}> <details class="newds newds-{{.Name}}" {{if eq $form.Pool .Name}}open{{end}}>
<summary>New dataset in {{.Name}}</summary> <summary>New dataset in {{.Name}}</summary>
<form method="post" action="/zfs/create" class="ds-form"> <form method="post" action="/zfs/create" class="ds-form">
@@ -46,6 +62,13 @@
<option value="off" {{if eq $form.Compression "off"}}selected{{end}}>off</option> <option value="off" {{if eq $form.Compression "off"}}selected{{end}}>off</option>
</select> </select>
</label> </label>
<label>Dedup
<select name="dedup">
<option value="" {{if eq $form.Dedup ""}}selected{{end}}>inherit</option>
<option value="on" {{if eq $form.Dedup "on"}}selected{{end}}>on</option>
<option value="off" {{if eq $form.Dedup "off"}}selected{{end}}>off</option>
</select>
</label>
<label>Quota <input name="quota" placeholder="none, or 100G / 1.5T" value="{{if eq $form.Pool .Name}}{{$form.Quota}}{{end}}"></label> <label>Quota <input name="quota" placeholder="none, or 100G / 1.5T" value="{{if eq $form.Pool .Name}}{{$form.Quota}}{{end}}"></label>
</div> </div>
<p class="small muted">Creates {{.Name}}/&lt;name&gt; (parents are created too), mounts it at /{{.Name}}/&lt;name&gt; and chowns the mountpoint to the owner.</p> <p class="small muted">Creates {{.Name}}/&lt;name&gt; (parents are created too), mounts it at /{{.Name}}/&lt;name&gt; and chowns the mountpoint to the owner.</p>
+44 -5
View File
@@ -68,12 +68,12 @@ func newTestServer(t *testing.T) (*Server, *sched.Scheduler, *store.Store) {
}, },
Pools: []collect.Pool{ Pools: []collect.Pool{
{Name: "tank", Health: "DEGRADED", Size: 8 << 40, Alloc: 3 << 40, Free: 5 << 40, CapPct: 37, FragPct: 5, Scan: "scrub repaired 0B", LastScrub: &old, {Name: "tank", Health: "DEGRADED", Size: 8 << 40, Alloc: 3 << 40, Free: 5 << 40, CapPct: 37, FragPct: 5, Scan: "scrub repaired 0B", LastScrub: &old,
Status: "One or more devices could not be opened.", Action: "Attach the missing device", Errors: "No known data errors", Status: "One or more devices could not be opened.", Action: "Attach the missing device", Errors: "No known data errors", DedupRatio: 1.42,
Vdevs: []collect.Vdev{{Name: "tank", State: "DEGRADED"}, {Name: "mirror-0", Depth: 1, State: "DEGRADED"}, {Name: "sda", Depth: 2, State: "ONLINE", Cksum: 2}, {Name: "123", Depth: 2, State: "UNAVAIL", Note: "was /dev/sdb1"}}}, Vdevs: []collect.Vdev{{Name: "tank", State: "DEGRADED"}, {Name: "mirror-0", Depth: 1, State: "DEGRADED"}, {Name: "sda", Depth: 2, State: "ONLINE", Cksum: 2}, {Name: "123", Depth: 2, State: "UNAVAIL", Note: "was /dev/sdb1"}}},
{Name: "tmp_pool", Health: "ONLINE", Size: 1 << 40, Alloc: 900 << 30, Free: 124 << 30, CapPct: 88, Scrubbing: true, Scan: "scrub in progress"}, {Name: "tmp_pool", Health: "ONLINE", Size: 1 << 40, Alloc: 900 << 30, Free: 124 << 30, CapPct: 88, Scrubbing: true, Scan: "scrub in progress"},
}, },
Datasets: []collect.Dataset{ Datasets: []collect.Dataset{
{Name: "tank", Used: 3 << 40, Avail: 5 << 40, Refer: 1 << 20, Mountpoint: "/tank"}, {Name: "tank", Used: 3 << 40, Avail: 5 << 40, Refer: 1 << 20, Mountpoint: "/tank", Dedup: "on"},
{Name: "tank/empty", Used: 96 << 10, Avail: 5 << 40, Refer: 96 << 10, Mountpoint: "/tank/empty"}, {Name: "tank/empty", Used: 96 << 10, Avail: 5 << 40, Refer: 96 << 10, Mountpoint: "/tank/empty"},
{Name: "tank/data", Used: 3 << 40, Avail: 5 << 40, Refer: 3 << 40, Mountpoint: "/tank/data"}, {Name: "tank/data", Used: 3 << 40, Avail: 5 << 40, Refer: 3 << 40, Mountpoint: "/tank/data"},
{Name: "tank/parent", Used: 96 << 10, Avail: 5 << 40, Refer: 96 << 10, Mountpoint: "/tank/parent"}, {Name: "tank/parent", Used: 96 << 10, Avail: 5 << 40, Refer: 96 << 10, Mountpoint: "/tank/parent"},
@@ -245,7 +245,7 @@ func TestZfsPageControls(t *testing.T) {
rec := httptest.NewRecorder() rec := httptest.NewRecorder()
srv.ServeHTTP(rec, httptest.NewRequest("GET", "/zfs?msg=hello", nil)) srv.ServeHTTP(rec, httptest.NewRequest("GET", "/zfs?msg=hello", nil))
body := rec.Body.String() body := rec.Body.String()
for _, want := range []string{"New dataset in tank", "New dataset in tmp_pool", "hello", `value="1000:1000"`, "type tank/empty to confirm"} { for _, want := range []string{"New dataset in tank", "New dataset in tmp_pool", "hello", `value="1000:1000"`, "type tank/empty to confirm", `action="/zfs/dedup"`, "Dedup for tmp_pool", "ratio 1.42x"} {
if !strings.Contains(body, want) { if !strings.Contains(body, want) {
t.Errorf("zfs page missing %q", want) t.Errorf("zfs page missing %q", want)
} }
@@ -274,6 +274,7 @@ func TestZfsCreate(t *testing.T) {
{"pool": {"tank"}, "name": {"x"}, "quota": {"lots"}}, // bad quota {"pool": {"tank"}, "name": {"x"}, "quota": {"lots"}}, // bad quota
{"pool": {"tank"}, "name": {"x"}, "owner": {"ansible"}}, // bad owner {"pool": {"tank"}, "name": {"x"}, "owner": {"ansible"}}, // bad owner
{"pool": {"tank"}, "name": {"x"}, "compression": {"gzip"}}, {"pool": {"tank"}, "name": {"x"}, "compression": {"gzip"}},
{"pool": {"tank"}, "name": {"x"}, "dedup": {"verify"}},
} }
for _, form := range bad { for _, form := range bad {
f.calls = nil f.calls = nil
@@ -289,11 +290,11 @@ func TestZfsCreate(t *testing.T) {
} }
} }
rec := postForm(srv, "/zfs/create", url.Values{"pool": {"tank"}, "name": {"/media/video/"}, "owner": {"1000:1000"}, "compression": {"lz4"}, "quota": {"100g"}}) rec := postForm(srv, "/zfs/create", url.Values{"pool": {"tank"}, "name": {"/media/video/"}, "owner": {"1000:1000"}, "compression": {"lz4"}, "dedup": {"on"}, "quota": {"100g"}})
if rec.Code != http.StatusSeeOther || !strings.Contains(rec.Header().Get("Location"), "created") { if rec.Code != http.StatusSeeOther || !strings.Contains(rec.Header().Get("Location"), "created") {
t.Fatalf("status %d location %q body %s", rec.Code, rec.Header().Get("Location"), rec.Body.String()) t.Fatalf("status %d location %q body %s", rec.Code, rec.Header().Get("Location"), rec.Body.String())
} }
want := "zfs create -p -o compression=lz4 -o quota=100G tank/media/video" want := "zfs create -p -o compression=lz4 -o dedup=on -o quota=100G tank/media/video"
if len(f.calls) != 3 || f.calls[0] != want || f.calls[2] != "chown 1000:1000 /tank/media/video" { if len(f.calls) != 3 || f.calls[0] != want || f.calls[2] != "chown 1000:1000 /tank/media/video" {
t.Errorf("calls = %v", f.calls) t.Errorf("calls = %v", f.calls)
} }
@@ -307,6 +308,44 @@ func TestZfsCreate(t *testing.T) {
} }
} }
func TestZfsDedup(t *testing.T) {
srv, _, _ := newTestServer(t)
f := &fakeRunner{}
srv.admin = zfsadm.New(f.run)
bad := []url.Values{
{"pool": {"nope"}, "dedup": {"on"}}, // unknown pool
{"pool": {"tank/data"}, "dedup": {"on"}}, // not a pool
{"pool": {"tank"}, "dedup": {"verify"}}, // unsupported value
{"pool": {"tank"}}, // missing value
}
for _, form := range bad {
f.calls = nil
rec := postForm(srv, "/zfs/dedup", form)
if rec.Code != 200 || !strings.Contains(rec.Body.String(), "Dedup not changed") {
t.Errorf("%v: status %d, no error shown", form, rec.Code)
}
if len(f.calls) != 0 {
t.Errorf("%v: commands ran: %v", form, f.calls)
}
}
rec := postForm(srv, "/zfs/dedup", url.Values{"pool": {"tmp_pool"}, "dedup": {"on"}})
if rec.Code != http.StatusSeeOther || !strings.Contains(rec.Header().Get("Location"), "#tmp_pool") {
t.Fatalf("status %d location %q", rec.Code, rec.Header().Get("Location"))
}
if len(f.calls) != 1 || f.calls[0] != "zfs set dedup=on tmp_pool" {
t.Errorf("calls = %v", f.calls)
}
f = &fakeRunner{fail: "zfs set"}
srv.admin = zfsadm.New(f.run)
rec = postForm(srv, "/zfs/dedup", url.Values{"pool": {"tank"}, "dedup": {"off"}})
if rec.Code != 200 || !strings.Contains(rec.Body.String(), "cannot do that") {
t.Errorf("failure not reported: %d", rec.Code)
}
}
func TestZfsDestroy(t *testing.T) { func TestZfsDestroy(t *testing.T) {
srv, _, _ := newTestServer(t) srv, _, _ := newTestServer(t)
f := &fakeRunner{} f := &fakeRunner{}
+35 -1
View File
@@ -1,5 +1,5 @@
// Package zfsadm performs the few mutating ZFS operations the UI offers // Package zfsadm performs the few mutating ZFS operations the UI offers
// (create / destroy datasets). Commands go through a Runner so they can be // (create / destroy datasets, toggle pool dedup). Commands go through a Runner so they can be
// executed on the host via systemd-run: nasmon runs in a sandboxed mount // executed on the host via systemd-run: nasmon runs in a sandboxed mount
// namespace (ProtectSystem=strict), where a plain "zfs create" would mount the // namespace (ProtectSystem=strict), where a plain "zfs create" would mount the
// new dataset only inside the service and leave the host without it. // new dataset only inside the service and leave the host without it.
@@ -59,6 +59,7 @@ func (a *Admin) ctx() (context.Context, context.CancelFunc) {
// CreateOpts are the user-visible knobs of "zfs create". // CreateOpts are the user-visible knobs of "zfs create".
type CreateOpts struct { type CreateOpts struct {
Compression string // "", lz4, zstd, off ("" = inherit) Compression string // "", lz4, zstd, off ("" = inherit)
Dedup string // "", on, off ("" = inherit)
Quota string // "", or a size like 100G / 1.5T Quota string // "", or a size like 100G / 1.5T
Owner string // uid:gid applied to the mountpoint; "" = leave root Owner string // uid:gid applied to the mountpoint; "" = leave root
} }
@@ -71,6 +72,9 @@ func (a *Admin) Create(full string, o CreateOpts) error {
if err := ValidateCompression(o.Compression); err != nil { if err := ValidateCompression(o.Compression); err != nil {
return err return err
} }
if err := ValidateDedup(o.Dedup); err != nil {
return err
}
if err := ValidateSize(o.Quota); err != nil { if err := ValidateSize(o.Quota); err != nil {
return err return err
} }
@@ -81,6 +85,9 @@ func (a *Admin) Create(full string, o CreateOpts) error {
if o.Compression != "" { if o.Compression != "" {
args = append(args, "-o", "compression="+o.Compression) args = append(args, "-o", "compression="+o.Compression)
} }
if o.Dedup != "" {
args = append(args, "-o", "dedup="+o.Dedup)
}
if o.Quota != "" { if o.Quota != "" {
args = append(args, "-o", "quota="+o.Quota) args = append(args, "-o", "quota="+o.Quota)
} }
@@ -106,6 +113,25 @@ func (a *Admin) Create(full string, o CreateOpts) error {
return nil return nil
} }
// SetDedup sets the dedup property (on/off) on the root dataset of pool, so it
// applies to every dataset that does not override it. Only newly written
// blocks are affected either way.
func (a *Admin) SetDedup(pool, value string) error {
if err := ValidateFullName(pool); err != nil {
return err
}
if strings.Contains(pool, "/") {
return errors.New("dedup is set on the pool root dataset only")
}
if value != "on" && value != "off" {
return errors.New("dedup must be on or off")
}
ctx, cancel := a.ctx()
defer cancel()
_, err := a.Run(ctx, "zfs", "set", "dedup="+value, pool)
return err
}
// EmptyRefer is the upper bound of "referenced" bytes for a dataset we still // EmptyRefer is the upper bound of "referenced" bytes for a dataset we still
// consider empty (a fresh dataset refers ~96-128 KiB of metadata). // consider empty (a fresh dataset refers ~96-128 KiB of metadata).
const EmptyRefer = 1 << 20 const EmptyRefer = 1 << 20
@@ -206,6 +232,14 @@ func ValidateCompression(s string) error {
return errors.New("compression must be inherit, lz4, zstd or off") return errors.New("compression must be inherit, lz4, zstd or off")
} }
func ValidateDedup(s string) error {
switch s {
case "", "on", "off":
return nil
}
return errors.New("dedup must be inherit, on or off")
}
func ValidateOwner(s string) error { func ValidateOwner(s string) error {
if s == "" || reOwner.MatchString(s) { if s == "" || reOwner.MatchString(s) {
return nil return nil
+35
View File
@@ -31,6 +31,9 @@ func TestValidators(t *testing.T) {
if ValidateCompression("gzip") == nil || ValidateCompression("lz4") != nil || ValidateCompression("") != nil { if ValidateCompression("gzip") == nil || ValidateCompression("lz4") != nil || ValidateCompression("") != nil {
t.Error("compression validation") t.Error("compression validation")
} }
if ValidateDedup("verify") == nil || ValidateDedup("on") != nil || ValidateDedup("") != nil {
t.Error("dedup validation")
}
if ValidateOwner("ansible") == nil || ValidateOwner("1000:1000") != nil || ValidateOwner("") != nil { if ValidateOwner("ansible") == nil || ValidateOwner("1000:1000") != nil || ValidateOwner("") != nil {
t.Error("owner validation") t.Error("owner validation")
} }
@@ -74,6 +77,16 @@ func TestCreate(t *testing.T) {
t.Errorf("calls:\n%s\nwant:\n%s", strings.Join(f.calls, "\n"), strings.Join(want, "\n")) t.Errorf("calls:\n%s\nwant:\n%s", strings.Join(f.calls, "\n"), strings.Join(want, "\n"))
} }
// dedup goes between compression and quota
f = &fake{}
a = New(f.run)
if err := a.Create("tank/d", CreateOpts{Compression: "zstd", Dedup: "on", Quota: "1T"}); err != nil {
t.Fatal(err)
}
if len(f.calls) != 1 || f.calls[0] != "zfs create -p -o compression=zstd -o dedup=on -o quota=1T tank/d" {
t.Errorf("calls = %v", f.calls)
}
// no owner, inherit everything // no owner, inherit everything
f = &fake{} f = &fake{}
a = New(f.run) a = New(f.run)
@@ -115,6 +128,28 @@ func TestCreate(t *testing.T) {
} }
} }
func TestSetDedup(t *testing.T) {
f := &fake{}
a := New(f.run)
for _, c := range [][2]string{{"tank/x", "on"}, {"tank", "verify"}, {"tank", ""}, {"-tank", "on"}, {"", "off"}} {
if err := a.SetDedup(c[0], c[1]); err == nil {
t.Errorf("%q=%q accepted", c[0], c[1])
}
}
if len(f.calls) != 0 {
t.Errorf("rejected input ran commands: %v", f.calls)
}
if err := a.SetDedup("tank", "on"); err != nil {
t.Fatal(err)
}
if err := a.SetDedup("tank", "off"); err != nil {
t.Fatal(err)
}
if strings.Join(f.calls, "|") != "zfs set dedup=on tank|zfs set dedup=off tank" {
t.Errorf("calls = %v", f.calls)
}
}
func TestDestroyEmpty(t *testing.T) { func TestDestroyEmpty(t *testing.T) {
ok := map[string]string{"zfs list -Hp -r -t filesystem": "tank/x\t98304", "zfs list -H -t snapshot": ""} ok := map[string]string{"zfs list -Hp -r -t filesystem": "tank/x\t98304", "zfs list -H -t snapshot": ""}
f := &fake{reply: ok} f := &fake{reply: ok}
+4 -1
View File
@@ -14,8 +14,11 @@ periodic SMART/ZFS checks with e-mail alerts.
The ZFS page (http://<host>:8090/zfs) can create datasets in a pool — name (nested The ZFS page (http://<host>:8090/zfs) can create datasets in a pool — name (nested
names like media/video are fine), owner uid:gid for the mountpoint, compression, names like media/video are fine), owner uid:gid for the mountpoint, compression,
optional quota — and destroy datasets that are empty (no data, no children, no dedup, optional quota — and destroy datasets that are empty (no data, no children, no
snapshots; the check is repeated against the live system before "zfs destroy"). snapshots; the check is repeated against the live system before "zfs destroy").
Each pool card also shows the dedup ratio and can turn dedup on/off on the pool
root dataset (`zfs set dedup=on|off <pool>`, inherited by child datasets; only new
writes are affected, and the dedup table needs a lot of RAM).
nasmon runs sandboxed (ProtectSystem=strict → own mount namespace), so these nasmon runs sandboxed (ProtectSystem=strict → own mount namespace), so these
commands are executed on the host through transient units: commands are executed on the host through transient units: