Skip to content

Commit eba21c0

Browse files
authored
summarize by hash id (#9)
* summarize by hash id * update readme
1 parent 5e3b6fb commit eba21c0

File tree

8 files changed

+1071
-53
lines changed

8 files changed

+1071
-53
lines changed

README.md

Lines changed: 40 additions & 24 deletions
Original file line numberDiff line numberDiff line change
@@ -32,36 +32,52 @@ $ trivy $IMAGENAME:$(dockertags -limit 1 -format json $IMAGENAME | jq -r .[0].ta
3232
## Examples
3333

3434
```bash
35-
$ dockertags goodwithtech/dockle
36-
+---------+-------+----------------------+-------------+
37-
| TAG | SIZE | CREATED AT | UPLOADED AT |
38-
+---------+-------+----------------------+-------------+
39-
| latest | 21.1M | 2019-12-16T14:05:18Z | NULL |
40-
| v0.2.4 | 21.1M | 2019-12-05T05:18:04Z | NULL |
41-
| v0.2.3 | 21.1M | 2019-11-17T15:03:10Z | NULL |
42-
| v0.2.2 | 21.1M | 2019-11-17T14:45:53Z | NULL |
43-
......
44-
| v0.0.18 | 20.4M | 2019-06-10T18:31:45Z | NULL |
45-
+---------+-------+----------------------+-------------+
35+
$ dockertags alpine
36+
+----------+------+----------------------+-------------+
37+
| TAG | SIZE | CREATED AT | UPLOADED AT |
38+
+----------+------+----------------------+-------------+
39+
| 3 | 2.7M | 2019-12-24T20:40:57Z | NULL |
40+
| 3.11 | | | |
41+
| latest | | | |
42+
| 3.11.2 | | | |
43+
+----------+------+----------------------+-------------+
44+
| edge | 2.7M | 2019-12-20T00:41:30Z | NULL |
45+
| 20191219 | | | |
46+
+----------+------+----------------------+-------------+
47+
| 3.11.0 | 2.7M | 2019-12-20T00:41:21Z | NULL |
48+
+----------+------+----------------------+-------------+
49+
| 20191114 | 2.7M | 2019-11-14T22:41:11Z | NULL |
50+
+----------+------+----------------------+-------------+
51+
| 3.10 | 2.7M | 2019-10-21T18:41:18Z | NULL |
52+
| 3.10.3 | | | |
53+
+----------+------+----------------------+-------------+
54+
| 20190925 | 2.7M | 2019-09-25T22:40:50Z | NULL |
55+
+----------+------+----------------------+-------------+
56+
| 3.10.2 | 2.7M | 2019-08-20T21:40:57Z | NULL |
57+
+----------+------+----------------------+-------------+
58+
| 3.8 | 2.1M | 2019-08-20T06:41:01Z | NULL |
59+
| 3.8.4 | | | |
60+
+----------+------+----------------------+-------------+
61+
| 20190809 | 2.7M | 2019-08-09T21:41:13Z | NULL |
62+
+----------+------+----------------------+-------------+
63+
| 3.10.1 | 2.7M | 2019-07-11T22:41:17Z | NULL |
64+
+----------+------+----------------------+-------------+
65+
66+
4667

4768
# You can set limit, filter and format
48-
$ dockertags -limit 2 -contain v -contain 2 -format json goodwithtech/dockle
69+
$ dockertags -limit 1 -contain latest -format json alpine
4970
[
5071
{
5172
"tags": [
52-
"v0.2.4"
53-
],
54-
"byte": 22154435,
55-
"created_at": "2019-12-05T05:18:04.174078Z",
56-
"uploaded_at": null
57-
},
58-
{
59-
"tags": [
60-
"v0.2.3"
73+
"latest",
74+
"3.11.2",
75+
"3.11",
76+
"3"
6177
],
62-
"byte": 22154435,
63-
"created_at": "2019-11-17T15:03:10.914092Z",
64-
"uploaded_at": null
78+
"byte": 2801778,
79+
"created_at": "2019-12-24T20:40:57.918177Z",
80+
"uploaded_at": "0001-01-01T00:00:00Z"
6581
}
6682
]
6783
```

go.mod

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -22,6 +22,7 @@ require (
2222
github.com/docker/libtrust v0.0.0-20160708172513-aabc10ec26b7 // indirect
2323
github.com/elazarl/goproxy v0.0.0-20190421051319-9d40249d3c2f // indirect
2424
github.com/elazarl/goproxy/ext v0.0.0-20190421051319-9d40249d3c2f // indirect
25+
github.com/google/go-cmp v0.3.1
2526
github.com/moul/http2curl v1.0.0 // indirect
2627
github.com/olekukonko/tablewriter v0.0.2-0.20190618033246-cc27d85e17ce
2728
github.com/opencontainers/go-digest v1.0.0-rc1

go.sum

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -78,6 +78,8 @@ github.com/google/go-cmp v0.2.0 h1:+dTQ8DZQJz0Mb/HjFlkptS1FeQ4cWSnN941F8aEG4SQ=
7878
github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M=
7979
github.com/google/go-cmp v0.3.0 h1:crn/baboCvb5fXaQ0IJ1SGTsTVrWpDsCWC8EGETZijY=
8080
github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
81+
github.com/google/go-cmp v0.3.1 h1:Xye71clBPdm5HgqGwUkwhbynsUJZhDbS20FvLhQ2izg=
82+
github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
8183
github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
8284
github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs=
8385
github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc=

internal/report/table.go

Lines changed: 6 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,7 @@ package report
33
import (
44
"io"
55
"os"
6+
"sort"
67
"strings"
78
"time"
89

@@ -21,14 +22,18 @@ func (w TableWriter) Write(tags types.ImageTags) (err error) {
2122
table.SetHeader([]string{"Tag", "Size", "Created At", "Uploaded At"})
2223

2324
for _, tag := range tags {
25+
targets := utils.StrByLen(tag.Tags)
26+
sort.Sort(targets)
27+
2428
table.Append([]string{
25-
strings.Join(tag.Tags, ","),
29+
strings.Join(targets, tablewriter.NEWLINE),
2630
getBytesize(tag.Byte),
2731
ttos(tag.CreatedAt),
2832
ttos(tag.UploadedAt),
2933
})
3034
}
3135
table.SetAlignment(tablewriter.ALIGN_LEFT)
36+
table.SetRowLine(true)
3237
table.Render()
3338

3439
return nil

internal/utils/strlen.go

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,15 @@
1+
package utils
2+
3+
type StrByLen []string
4+
5+
func (a StrByLen) Len() int {
6+
return len(a)
7+
}
8+
9+
func (a StrByLen) Less(i, j int) bool {
10+
return len(a[i]) < len(a[j])
11+
}
12+
13+
func (a StrByLen) Swap(i, j int) {
14+
a[i], a[j] = a[j], a[i]
15+
}

pkg/provider/dockerhub/dockerhub.go

Lines changed: 75 additions & 28 deletions
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,7 @@ package dockerhub
33
import (
44
"context"
55
"fmt"
6+
"sort"
67

78
"golang.org/x/sync/errgroup"
89

@@ -32,6 +33,19 @@ type ImageSummary struct {
3233
Name string `json:"name"`
3334
FullSize int `json:"full_size"`
3435
LastUpdated string `json:"last_updated"`
36+
Images Images `json:"images"`
37+
}
38+
39+
type Images []Image
40+
type Image struct {
41+
Digest string `json:"digest"`
42+
Architecture string `json:"architecture"`
43+
}
44+
45+
func (t Images) Len() int { return len(t) }
46+
func (t Images) Swap(i, j int) { t[i], t[j] = t[j], t[i] }
47+
func (t Images) Less(i, j int) bool {
48+
return (t[i].Digest) > (t[j].Digest)
3549
}
3650

3751
func (p *DockerHub) Run(ctx context.Context, domain, repository string, reqOpt *types.RequestOption, filterOpt *types.FilterOption) (types.ImageTags, error) {
@@ -46,14 +60,18 @@ func (p *DockerHub) Run(ctx context.Context, domain, repository string, reqOpt *
4660
if err != nil {
4761
return nil, err
4862
}
49-
imageTags := p.convertResultToTag(tagResp.Results)
50-
if reqOpt.MaxCount > 0 && len(imageTags) > reqOpt.MaxCount {
51-
return imageTags, nil
52-
}
63+
// imageTags := p.convertResultToTag(tagResp.Results)
64+
// if reqOpt.MaxCount > 0 && len(imageTags) > reqOpt.MaxCount {
65+
// return imageTags, nil
66+
// }
67+
68+
// create all in one []ImageSummary
69+
totalTagSummary := tagResp.Results
5370

5471
lastPage := calcMaxRequestPage(tagResp.Count, reqOpt.MaxCount, filterOpt)
5572
// create ch (page - 1), already fetched first page,
56-
tagsPerPage := make(chan types.ImageTags, lastPage-1)
73+
//tagsPerPage := make(chan types.ImageTags, lastPage-1)
74+
tagsPerPage := make(chan []ImageSummary, lastPage-1)
5775
eg := errgroup.Group{}
5876
for page := 2; page <= lastPage; page++ {
5977
page := page
@@ -62,7 +80,7 @@ func (p *DockerHub) Run(ctx context.Context, domain, repository string, reqOpt *
6280
if err != nil {
6381
return err
6482
}
65-
tagsPerPage <- p.convertResultToTag(tagResp.Results)
83+
tagsPerPage <- tagResp.Results
6684
return nil
6785
})
6886
}
@@ -73,32 +91,59 @@ func (p *DockerHub) Run(ctx context.Context, domain, repository string, reqOpt *
7391
for page := 2; page <= lastPage; page++ {
7492
select {
7593
case tags := <-tagsPerPage:
76-
imageTags = append(imageTags, tags...)
94+
totalTagSummary = append(totalTagSummary, tags...)
7795
}
7896
}
79-
return imageTags, nil
97+
return p.convertResultToTag(totalTagSummary), nil
8098
}
8199

82100
func (p *DockerHub) convertResultToTag(summaries []ImageSummary) types.ImageTags {
83-
tags := []types.ImageTag{}
84-
for _, detail := range summaries {
85-
if detail.Name == "" {
101+
// TODO : refactor it
102+
103+
// create map : key is image hash
104+
pools := map[string]types.ImageTag{}
105+
for _, imageSummary := range summaries {
106+
if imageSummary.Name == "" {
107+
log.Logger.Debugf("no tag data :%v", imageSummary)
86108
continue
87109
}
88-
createdAt, _ := time.Parse(time.RFC3339Nano, detail.LastUpdated)
89-
tagNames := []string{detail.Name}
90-
if !utils.MatchConditionTags(p.filterOpt, tagNames) {
110+
if len(imageSummary.Images) == 0 {
111+
log.Logger.Debugf("no image layer data :%v", imageSummary)
91112
continue
92113
}
93-
tags = append(tags, types.ImageTag{
94-
Tags: tagNames,
95-
Byte: detail.FullSize,
96-
CreatedAt: createdAt,
97-
})
114+
sort.Sort(imageSummary.Images)
115+
firstHash := imageSummary.Images[0].Digest
116+
target, ok := pools[firstHash]
117+
// create first one if not exist
118+
if !ok {
119+
pools[firstHash] = createImageTag(imageSummary)
120+
continue
121+
}
122+
// update exist ImageTag
123+
target.Tags = append(target.Tags, imageSummary.Name)
124+
pools[firstHash] = target
125+
}
126+
127+
tags := []types.ImageTag{}
128+
for _, imageTag := range pools {
129+
if !utils.MatchConditionTags(p.filterOpt, imageTag.Tags) {
130+
continue
131+
}
132+
tags = append(tags, imageTag)
98133
}
99134
return tags
100135
}
101136

137+
func createImageTag(is ImageSummary) types.ImageTag {
138+
createdAt, _ := time.Parse(time.RFC3339Nano, is.LastUpdated)
139+
tagNames := []string{is.Name}
140+
return types.ImageTag{
141+
Tags: tagNames,
142+
Byte: is.FullSize,
143+
CreatedAt: createdAt,
144+
}
145+
}
146+
102147
// getTagResponse returns the tags for a specific repository.
103148
// curl 'https://registry.hub.docker.com/v2/repositories/library/debian/tags/'
104149
func getTagResponse(ctx context.Context, auth dockertypes.AuthConfig, timeout time.Duration, repository string, page int) (tagsResponse, error) {
@@ -113,13 +158,15 @@ func getTagResponse(ctx context.Context, auth dockertypes.AuthConfig, timeout ti
113158
}
114159

115160
func calcMaxRequestPage(totalCnt, needCnt int, option *types.FilterOption) int {
116-
maxPage := totalCnt/types.ImagePerPage + 1
117-
if needCnt == 0 || len(option.Contain) != 0 {
118-
return maxPage
119-
}
120-
needPage := needCnt/types.ImagePerPage + 1
121-
if needPage >= maxPage {
122-
return maxPage
123-
}
124-
return needPage
161+
// TODO : currently always fetch all pages for show alias
162+
return totalCnt/types.ImagePerPage + 1
163+
// maxPage := totalCnt/types.ImagePerPage + 1
164+
// if needCnt == 0 || len(option.Contain) != 0 {
165+
// return maxPage
166+
// }
167+
// needPage := needCnt/types.ImagePerPage + 1
168+
// if needPage >= maxPage {
169+
// return maxPage
170+
// }
171+
// return needPage
125172
}
Lines changed: 85 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,85 @@
1+
package dockerhub
2+
3+
import (
4+
"encoding/json"
5+
"io/ioutil"
6+
"sort"
7+
"testing"
8+
9+
"github.com/google/go-cmp/cmp"
10+
"github.com/google/go-cmp/cmp/cmpopts"
11+
12+
"github.com/goodwithtech/dockertags/internal/types"
13+
)
14+
15+
func TestScanImage(t *testing.T) {
16+
testcases := map[string]struct {
17+
filePath string
18+
filterOpt types.FilterOption
19+
expected types.ImageTags
20+
}{
21+
"debian page1": {
22+
filePath: "./testdata/page1.json",
23+
filterOpt: types.FilterOption{},
24+
expected: types.ImageTags{
25+
types.ImageTag{
26+
Tags: []string{"unstable-slim", "unstable-20191118-slim"},
27+
},
28+
types.ImageTag{
29+
Tags: []string{"unstable", "unstable-20191118"},
30+
},
31+
types.ImageTag{
32+
Tags: []string{"testing-slim", "testing-20191118-slim"},
33+
},
34+
types.ImageTag{
35+
Tags: []string{"testing-backports"},
36+
},
37+
types.ImageTag{
38+
Tags: []string{"testing", "testing-20191118"},
39+
},
40+
types.ImageTag{
41+
Tags: []string{"stretch-slim"},
42+
},
43+
},
44+
},
45+
"debian filter slim": {
46+
filePath: "./testdata/page1.json",
47+
filterOpt: types.FilterOption{Contain: []string{"slim"}},
48+
expected: types.ImageTags{
49+
types.ImageTag{
50+
Tags: []string{"unstable-slim", "unstable-20191118-slim"},
51+
},
52+
types.ImageTag{
53+
Tags: []string{"testing-slim", "testing-20191118-slim"},
54+
},
55+
types.ImageTag{
56+
Tags: []string{"stretch-slim"},
57+
},
58+
},
59+
},
60+
}
61+
62+
for tc, v := range testcases {
63+
dockerHub := DockerHub{filterOpt: &v.filterOpt}
64+
var data tagsResponse
65+
file, err := ioutil.ReadFile(v.filePath)
66+
if err != nil {
67+
t.Errorf("readfile error: %w", err)
68+
continue
69+
}
70+
json.Unmarshal(file, &data)
71+
actual := dockerHub.convertResultToTag(data.Results)
72+
opts := []cmp.Option{
73+
cmp.Transformer("Sort", func(in []string) []string {
74+
out := append([]string{}, in...) // Copy input to avoid mutating it
75+
sort.Strings(out)
76+
return out
77+
}),
78+
cmpopts.IgnoreFields(types.ImageTag{}, "Byte", "CreatedAt"),
79+
}
80+
sort.Sort(actual)
81+
if diff := cmp.Diff(v.expected, actual, opts...); diff != "" {
82+
t.Errorf("%s: diff %v", tc, diff)
83+
}
84+
}
85+
}

0 commit comments

Comments
 (0)