From 09f428ea76f2eea13e39f363c9838a94c74990d5 Mon Sep 17 00:00:00 2001 From: wuhuizuo Date: Wed, 18 Sep 2024 22:01:31 +0800 Subject: [PATCH] feat(dl): support download file with `file_regex` param (#169) example: curl -v "http://localhost:8000/oci-file/hub.pingcap.net/pingcap/tidb/package?tag=v8.3.0_linux_amd64& file_regex=tidb-lightning-.%2A%5B.%5Dsha256" to download the file with regex pattern `tidb-lightning-.*[.]sha256`. Signed-off-by: wuhuizuo Signed-off-by: wuhuizuo --- dl/Dockerfile | 2 +- dl/design/design.go | 24 ++++++-- dl/gen/http/cli/server/cli.go | 22 +++---- dl/gen/http/oci/client/cli.go | 27 +++++++-- dl/gen/http/oci/client/encode_decode.go | 7 ++- dl/gen/http/oci/server/encode_decode.go | 20 +++++-- dl/gen/http/oci/server/types.go | 5 +- dl/gen/http/openapi.json | 2 +- dl/gen/http/openapi.yaml | 16 +++-- dl/gen/http/openapi3.json | 2 +- dl/gen/http/openapi3.yaml | 78 ++++++++++++++----------- dl/gen/oci/service.go | 4 +- dl/go.mod | 24 ++++---- dl/go.sum | 43 +++++--------- dl/oci.go | 38 +++++++++++- 15 files changed, 198 insertions(+), 116 deletions(-) diff --git a/dl/Dockerfile b/dl/Dockerfile index e47fddc4..6319d073 100644 --- a/dl/Dockerfile +++ b/dl/Dockerfile @@ -1,4 +1,4 @@ -FROM golang:1.22.4 AS builder +FROM golang:1.23.1 AS builder COPY . /app RUN --mount=type=cache,target=/go/pkg/mod cd /app && go build -o server ./cmd/server diff --git a/dl/design/design.go b/dl/design/design.go index d29b6297..d9e4efa6 100644 --- a/dl/design/design.go +++ b/dl/design/design.go @@ -55,10 +55,21 @@ var _ = Service("oci", func() { Method("download-file", func() { Payload(func() { - Field(1, "repository", String, "OCI artifact repository") - Field(2, "tag", String, "OCI artifact tag") - Field(3, "file", String, "file name in OCI artifact") - Required("repository", "tag", "file") + Field(1, "repository", String, "OCI artifact repository", func() { + Example("hub.pingcap.net/pingcap/tidb/package") + }) + Field(2, "tag", String, "OCI artifact tag", func() { + Example("v8.1.0_darwin_arm64") + }) + Field(3, "file", String, "file name in OCI artifact", func() { + Example("tidb-v7.5.0-darwin-arm64.tar.gz") + }) + Field(4, "file_regex", String, "file name regexp pattern", func() { + Format(FormatRegexp) + Example("tidb-.+[.]tar[.]gz") + }) + + Required("repository", "tag") }) // The use of Result here illustrates how HTTP headers can still be @@ -79,8 +90,9 @@ var _ = Service("oci", func() { HTTP(func() { GET("/oci-file/{*repository}") - Param("file:file", String, "file name in OCI artifact") - Param("tag:tag", String, "OCI artifact tag") + Param("tag", String, "OCI artifact tag") + Param("file", String, "file name in OCI artifact") + Param("file_regex", String, "file name regex pattern in OCI artifact") // Bypass response body encoder code generation to alleviate need for // loading the entire response body in memory. diff --git a/dl/gen/http/cli/server/cli.go b/dl/gen/http/cli/server/cli.go index 681b1077..cc10be7b 100644 --- a/dl/gen/http/cli/server/cli.go +++ b/dl/gen/http/cli/server/cli.go @@ -33,8 +33,8 @@ ks3 download-object // UsageExamples produces an example of a valid invocation of the CLI tool. func UsageExamples() string { return os.Args[0] + ` health healthz` + "\n" + - os.Args[0] + ` oci list-files --repository "Earum quia." --tag "Nostrum cupiditate ipsa explicabo cum a."` + "\n" + - os.Args[0] + ` ks3 download-object --bucket "Quos quaerat nulla repudiandae magnam vel nobis." --key "At non dignissimos error et repellendus."` + "\n" + + os.Args[0] + ` oci list-files --repository "Excepturi perferendis dolores voluptas eius non." --tag "Est reprehenderit quibusdam eveniet velit."` + "\n" + + os.Args[0] + ` ks3 download-object --bucket "Reiciendis eligendi magnam officiis recusandae est fugiat." --key "Provident temporibus occaecati unde."` + "\n" + "" } @@ -62,8 +62,9 @@ func ParseEndpoint( ociDownloadFileFlags = flag.NewFlagSet("download-file", flag.ExitOnError) ociDownloadFileRepositoryFlag = ociDownloadFileFlags.String("repository", "REQUIRED", "OCI artifact repository") - ociDownloadFileFileFlag = ociDownloadFileFlags.String("file", "REQUIRED", "") ociDownloadFileTagFlag = ociDownloadFileFlags.String("tag", "REQUIRED", "") + ociDownloadFileFileFlag = ociDownloadFileFlags.String("file", "", "") + ociDownloadFileFileRegexFlag = ociDownloadFileFlags.String("file-regex", "", "") ociDownloadFileSha256Flags = flag.NewFlagSet("download-file-sha256", flag.ExitOnError) ociDownloadFileSha256RepositoryFlag = ociDownloadFileSha256Flags.String("repository", "REQUIRED", "OCI artifact repository") @@ -192,7 +193,7 @@ func ParseEndpoint( data, err = ocic.BuildListFilesPayload(*ociListFilesRepositoryFlag, *ociListFilesTagFlag) case "download-file": endpoint = c.DownloadFile() - data, err = ocic.BuildDownloadFilePayload(*ociDownloadFileRepositoryFlag, *ociDownloadFileFileFlag, *ociDownloadFileTagFlag) + data, err = ocic.BuildDownloadFilePayload(*ociDownloadFileRepositoryFlag, *ociDownloadFileTagFlag, *ociDownloadFileFileFlag, *ociDownloadFileFileRegexFlag) case "download-file-sha256": endpoint = c.DownloadFileSha256() data, err = ocic.BuildDownloadFileSha256Payload(*ociDownloadFileSha256RepositoryFlag, *ociDownloadFileSha256FileFlag, *ociDownloadFileSha256TagFlag) @@ -270,20 +271,21 @@ ListFiles implements list-files. -tag STRING: Example: - %[1]s oci list-files --repository "Earum quia." --tag "Nostrum cupiditate ipsa explicabo cum a." + %[1]s oci list-files --repository "Excepturi perferendis dolores voluptas eius non." --tag "Est reprehenderit quibusdam eveniet velit." `, os.Args[0]) } func ociDownloadFileUsage() { - fmt.Fprintf(os.Stderr, `%[1]s [flags] oci download-file -repository STRING -file STRING -tag STRING + fmt.Fprintf(os.Stderr, `%[1]s [flags] oci download-file -repository STRING -tag STRING -file STRING -file-regex STRING DownloadFile implements download-file. -repository STRING: OCI artifact repository - -file STRING: -tag STRING: + -file STRING: + -file-regex STRING: Example: - %[1]s oci download-file --repository "Quis maiores hic et commodi aut." --file "Corrupti qui qui iusto." --tag "Enim animi exercitationem voluptate perferendis ut." + %[1]s oci download-file --repository "hub.pingcap.net/pingcap/tidb/package" --tag "v8.1.0_darwin_arm64" --file "tidb-v7.5.0-darwin-arm64.tar.gz" --file-regex "tidb-.+[.]tar[.]gz" `, os.Args[0]) } @@ -296,7 +298,7 @@ DownloadFileSha256 implements download-file-sha256. -tag STRING: Example: - %[1]s oci download-file-sha256 --repository "Omnis est natus exercitationem aliquid tempora cumque." --file "Reiciendis eligendi magnam officiis recusandae est fugiat." --tag "Provident temporibus occaecati unde." + %[1]s oci download-file-sha256 --repository "Quis maiores hic et commodi aut." --file "Corrupti qui qui iusto." --tag "Enim animi exercitationem voluptate perferendis ut." `, os.Args[0]) } @@ -321,6 +323,6 @@ DownloadObject implements download-object. -key STRING: object key Example: - %[1]s ks3 download-object --bucket "Quos quaerat nulla repudiandae magnam vel nobis." --key "At non dignissimos error et repellendus." + %[1]s ks3 download-object --bucket "Reiciendis eligendi magnam officiis recusandae est fugiat." --key "Provident temporibus occaecati unde." `, os.Args[0]) } diff --git a/dl/gen/http/oci/client/cli.go b/dl/gen/http/oci/client/cli.go index ce8bbbab..a0bbfe06 100644 --- a/dl/gen/http/oci/client/cli.go +++ b/dl/gen/http/oci/client/cli.go @@ -9,6 +9,7 @@ package client import ( oci "github.com/PingCAP-QE/ee-apps/dl/gen/oci" + goa "goa.design/goa/v3/pkg" ) // BuildListFilesPayload builds the payload for the oci list-files endpoint @@ -31,23 +32,37 @@ func BuildListFilesPayload(ociListFilesRepository string, ociListFilesTag string // BuildDownloadFilePayload builds the payload for the oci download-file // endpoint from CLI flags. -func BuildDownloadFilePayload(ociDownloadFileRepository string, ociDownloadFileFile string, ociDownloadFileTag string) (*oci.DownloadFilePayload, error) { +func BuildDownloadFilePayload(ociDownloadFileRepository string, ociDownloadFileTag string, ociDownloadFileFile string, ociDownloadFileFileRegex string) (*oci.DownloadFilePayload, error) { + var err error var repository string { repository = ociDownloadFileRepository } - var file string - { - file = ociDownloadFileFile - } var tag string { tag = ociDownloadFileTag } + var file *string + { + if ociDownloadFileFile != "" { + file = &ociDownloadFileFile + } + } + var fileRegex *string + { + if ociDownloadFileFileRegex != "" { + fileRegex = &ociDownloadFileFileRegex + err = goa.MergeErrors(err, goa.ValidateFormat("file_regex", *fileRegex, goa.FormatRegexp)) + if err != nil { + return nil, err + } + } + } v := &oci.DownloadFilePayload{} v.Repository = repository - v.File = file v.Tag = tag + v.File = file + v.FileRegex = fileRegex return v, nil } diff --git a/dl/gen/http/oci/client/encode_decode.go b/dl/gen/http/oci/client/encode_decode.go index 5cec0400..75d54d95 100644 --- a/dl/gen/http/oci/client/encode_decode.go +++ b/dl/gen/http/oci/client/encode_decode.go @@ -129,8 +129,13 @@ func EncodeDownloadFileRequest(encoder func(*http.Request) goahttp.Encoder) func return goahttp.ErrInvalidType("oci", "download-file", "*oci.DownloadFilePayload", v) } values := req.URL.Query() - values.Add("file", p.File) values.Add("tag", p.Tag) + if p.File != nil { + values.Add("file", *p.File) + } + if p.FileRegex != nil { + values.Add("file_regex", *p.FileRegex) + } req.URL.RawQuery = values.Encode() return nil } diff --git a/dl/gen/http/oci/server/encode_decode.go b/dl/gen/http/oci/server/encode_decode.go index 5a834672..4e920c9f 100644 --- a/dl/gen/http/oci/server/encode_decode.go +++ b/dl/gen/http/oci/server/encode_decode.go @@ -77,26 +77,34 @@ func DecodeDownloadFileRequest(mux goahttp.Muxer, decoder func(*http.Request) go return func(r *http.Request) (any, error) { var ( repository string - file string tag string + file *string + fileRegex *string err error params = mux.Vars(r) ) repository = params["repository"] qp := r.URL.Query() - file = qp.Get("file") - if file == "" { - err = goa.MergeErrors(err, goa.MissingFieldError("file", "query string")) - } tag = qp.Get("tag") if tag == "" { err = goa.MergeErrors(err, goa.MissingFieldError("tag", "query string")) } + fileRaw := qp.Get("file") + if fileRaw != "" { + file = &fileRaw + } + fileRegexRaw := qp.Get("file_regex") + if fileRegexRaw != "" { + fileRegex = &fileRegexRaw + } + if fileRegex != nil { + err = goa.MergeErrors(err, goa.ValidateFormat("file_regex", *fileRegex, goa.FormatRegexp)) + } if err != nil { return nil, err } - payload := NewDownloadFilePayload(repository, file, tag) + payload := NewDownloadFilePayload(repository, tag, file, fileRegex) return payload, nil } diff --git a/dl/gen/http/oci/server/types.go b/dl/gen/http/oci/server/types.go index 2ea9a880..43aaf336 100644 --- a/dl/gen/http/oci/server/types.go +++ b/dl/gen/http/oci/server/types.go @@ -21,11 +21,12 @@ func NewListFilesPayload(repository string, tag string) *oci.ListFilesPayload { } // NewDownloadFilePayload builds a oci service download-file endpoint payload. -func NewDownloadFilePayload(repository string, file string, tag string) *oci.DownloadFilePayload { +func NewDownloadFilePayload(repository string, tag string, file *string, fileRegex *string) *oci.DownloadFilePayload { v := &oci.DownloadFilePayload{} v.Repository = repository - v.File = file v.Tag = tag + v.File = file + v.FileRegex = fileRegex return v } diff --git a/dl/gen/http/openapi.json b/dl/gen/http/openapi.json index 69e302b1..3fcd8e9a 100644 --- a/dl/gen/http/openapi.json +++ b/dl/gen/http/openapi.json @@ -1 +1 @@ -{"swagger":"2.0","info":{"title":"Download OCI artifacts Service","description":"Service for downloading files from OCI artifact","version":"0.0.1"},"host":"localhost:8000","consumes":["application/json","application/xml","application/gob"],"produces":["application/json","application/xml","application/gob"],"paths":{"/healthz":{"get":{"tags":["health"],"summary":"healthz health","operationId":"health#healthz","responses":{"200":{"description":"OK response.","schema":{"type":"boolean"}}},"schemes":["http"]}},"/livez":{"get":{"tags":["health"],"summary":"livez health","operationId":"health#livez","responses":{"200":{"description":"OK response.","schema":{"type":"boolean"}}},"schemes":["http"]}},"/oci-file-sha256/{repository}":{"get":{"tags":["oci"],"summary":"download-file-sha256 oci","operationId":"oci#download-file-sha256","produces":["application/plain-text"],"parameters":[{"name":"file","in":"query","description":"file name in OCI artifact","required":true,"type":"string"},{"name":"tag","in":"query","description":"OCI artifact tag","required":true,"type":"string"},{"name":"repository","in":"path","description":"OCI artifact repository","required":true,"type":"string"}],"responses":{"200":{"description":"OK response.","headers":{"Content-Disposition":{"description":"Content-Disposition header for downloading","type":"string"},"Content-Length":{"description":"Length is the downloaded content length in bytes.","type":"int64"}}}},"schemes":["http"]}},"/oci-file/{repository}":{"get":{"tags":["oci"],"summary":"download-file oci","operationId":"oci#download-file","produces":["application/octet-stream"],"parameters":[{"name":"file","in":"query","description":"file name in OCI artifact","required":true,"type":"string"},{"name":"tag","in":"query","description":"OCI artifact tag","required":true,"type":"string"},{"name":"repository","in":"path","description":"OCI artifact repository","required":true,"type":"string"}],"responses":{"200":{"description":"OK response.","headers":{"Content-Disposition":{"description":"Content-Disposition header for downloading","type":"string"},"Content-Length":{"description":"Length is the downloaded content length in bytes.","type":"int64"}}}},"schemes":["http"]}},"/oci-files/{repository}":{"get":{"tags":["oci"],"summary":"list-files oci","operationId":"oci#list-files","parameters":[{"name":"tag","in":"query","description":"OCI artifact tag","required":true,"type":"string"},{"name":"repository","in":"path","description":"OCI artifact repository","required":true,"type":"string"}],"responses":{"200":{"description":"OK response.","schema":{"type":"array","items":{"type":"string","example":"Nam quibusdam recusandae dolor voluptas."}}}},"schemes":["http"]}},"/s3-obj/{bucket}/{key}":{"get":{"tags":["ks3"],"summary":"download-object ks3","operationId":"ks3#download-object","produces":["application/octet-stream"],"parameters":[{"name":"bucket","in":"path","description":"bucket name","required":true,"type":"string"},{"name":"key","in":"path","description":"object key","required":true,"type":"string"}],"responses":{"200":{"description":"OK response.","headers":{"Content-Disposition":{"description":"Content-Disposition header for downloading","type":"string"},"Content-Length":{"description":"Length is the downloaded content length in bytes.","type":"int64"}}}},"schemes":["http"]}}}} \ No newline at end of file +{"swagger":"2.0","info":{"title":"Download OCI artifacts Service","description":"Service for downloading files from OCI artifact","version":"0.0.1"},"host":"localhost:8000","consumes":["application/json","application/xml","application/gob"],"produces":["application/json","application/xml","application/gob"],"paths":{"/healthz":{"get":{"tags":["health"],"summary":"healthz health","operationId":"health#healthz","responses":{"200":{"description":"OK response.","schema":{"type":"boolean"}}},"schemes":["http"]}},"/livez":{"get":{"tags":["health"],"summary":"livez health","operationId":"health#livez","responses":{"200":{"description":"OK response.","schema":{"type":"boolean"}}},"schemes":["http"]}},"/oci-file-sha256/{repository}":{"get":{"tags":["oci"],"summary":"download-file-sha256 oci","operationId":"oci#download-file-sha256","produces":["application/plain-text"],"parameters":[{"name":"file","in":"query","description":"file name in OCI artifact","required":true,"type":"string"},{"name":"tag","in":"query","description":"OCI artifact tag","required":true,"type":"string"},{"name":"repository","in":"path","description":"OCI artifact repository","required":true,"type":"string"}],"responses":{"200":{"description":"OK response.","headers":{"Content-Disposition":{"description":"Content-Disposition header for downloading","type":"string"},"Content-Length":{"description":"Length is the downloaded content length in bytes.","type":"int64"}}}},"schemes":["http"]}},"/oci-file/{repository}":{"get":{"tags":["oci"],"summary":"download-file oci","operationId":"oci#download-file","produces":["application/octet-stream"],"parameters":[{"name":"tag","in":"query","description":"OCI artifact tag","required":true,"type":"string"},{"name":"file","in":"query","description":"file name in OCI artifact","required":false,"type":"string"},{"name":"file_regex","in":"query","description":"file name regex pattern in OCI artifact","required":false,"type":"string","format":"regexp"},{"name":"repository","in":"path","description":"OCI artifact repository","required":true,"type":"string"}],"responses":{"200":{"description":"OK response.","headers":{"Content-Disposition":{"description":"Content-Disposition header for downloading","type":"string"},"Content-Length":{"description":"Length is the downloaded content length in bytes.","type":"int64"}}}},"schemes":["http"]}},"/oci-files/{repository}":{"get":{"tags":["oci"],"summary":"list-files oci","operationId":"oci#list-files","parameters":[{"name":"tag","in":"query","description":"OCI artifact tag","required":true,"type":"string"},{"name":"repository","in":"path","description":"OCI artifact repository","required":true,"type":"string"}],"responses":{"200":{"description":"OK response.","schema":{"type":"array","items":{"type":"string","example":"Et aut labore veniam et maiores qui."}}}},"schemes":["http"]}},"/s3-obj/{bucket}/{key}":{"get":{"tags":["ks3"],"summary":"download-object ks3","operationId":"ks3#download-object","produces":["application/octet-stream"],"parameters":[{"name":"bucket","in":"path","description":"bucket name","required":true,"type":"string"},{"name":"key","in":"path","description":"object key","required":true,"type":"string"}],"responses":{"200":{"description":"OK response.","headers":{"Content-Disposition":{"description":"Content-Disposition header for downloading","type":"string"},"Content-Length":{"description":"Length is the downloaded content length in bytes.","type":"int64"}}}},"schemes":["http"]}}}} \ No newline at end of file diff --git a/dl/gen/http/openapi.yaml b/dl/gen/http/openapi.yaml index edc1961a..8415ed65 100644 --- a/dl/gen/http/openapi.yaml +++ b/dl/gen/http/openapi.yaml @@ -84,16 +84,22 @@ paths: produces: - application/octet-stream parameters: + - name: tag + in: query + description: OCI artifact tag + required: true + type: string - name: file in: query description: file name in OCI artifact - required: true + required: false type: string - - name: tag + - name: file_regex in: query - description: OCI artifact tag - required: true + description: file name regex pattern in OCI artifact + required: false type: string + format: regexp - name: repository in: path description: OCI artifact repository @@ -135,7 +141,7 @@ paths: type: array items: type: string - example: Nam quibusdam recusandae dolor voluptas. + example: Et aut labore veniam et maiores qui. schemes: - http /s3-obj/{bucket}/{key}: diff --git a/dl/gen/http/openapi3.json b/dl/gen/http/openapi3.json index ddcc665a..4df198cb 100644 --- a/dl/gen/http/openapi3.json +++ b/dl/gen/http/openapi3.json @@ -1 +1 @@ -{"openapi":"3.0.3","info":{"title":"Download OCI artifacts Service","description":"Service for downloading files from OCI artifact","version":"0.0.1"},"servers":[{"url":"http://localhost:8000"}],"paths":{"/healthz":{"get":{"tags":["health"],"summary":"healthz health","operationId":"health#healthz","responses":{"200":{"description":"OK response.","content":{"application/json":{"schema":{"type":"boolean","example":true},"example":false}}}}}},"/livez":{"get":{"tags":["health"],"summary":"livez health","operationId":"health#livez","responses":{"200":{"description":"OK response.","content":{"application/json":{"schema":{"type":"boolean","example":true},"example":false}}}}}},"/oci-file-sha256/{repository}":{"get":{"tags":["oci"],"summary":"download-file-sha256 oci","operationId":"oci#download-file-sha256","parameters":[{"name":"file","in":"query","description":"file name in OCI artifact","allowEmptyValue":true,"required":true,"schema":{"type":"string","description":"file name in OCI artifact","example":"Vero voluptatum perspiciatis omnis qui vel."},"example":"A at dolor soluta molestiae."},{"name":"tag","in":"query","description":"OCI artifact tag","allowEmptyValue":true,"required":true,"schema":{"type":"string","description":"OCI artifact tag","example":"Sequi dolor voluptatem atque."},"example":"Nesciunt eum quia."},{"name":"repository","in":"path","description":"OCI artifact repository","required":true,"schema":{"type":"string","description":"OCI artifact repository","example":"Nobis possimus eaque dolor sunt similique."},"example":"Quod nulla nostrum voluptatem soluta reprehenderit reiciendis."}],"responses":{"200":{"description":"OK response.","headers":{"Content-Disposition":{"description":"Content-Disposition header for downloading","schema":{"type":"string","description":"Content-Disposition header for downloading","example":"attachment; filename*=UTF-8''tidb-v7.5.0-darwin-arm64.tar.gz.sha256"},"example":"attachment; filename*=UTF-8''tidb-v7.5.0-darwin-arm64.tar.gz.sha256"},"Content-Length":{"description":"Length is the downloaded content length in bytes.","schema":{"type":"integer","description":"Length is the downloaded content length in bytes.","example":4194304,"format":"int64"},"example":4194304}},"content":{"application/plain-text":{"schema":{"type":"string","format":"binary"}}}}}}},"/oci-file/{repository}":{"get":{"tags":["oci"],"summary":"download-file oci","operationId":"oci#download-file","parameters":[{"name":"file","in":"query","description":"file name in OCI artifact","allowEmptyValue":true,"required":true,"schema":{"type":"string","description":"file name in OCI artifact","example":"Est et."},"example":"Non magnam a corporis."},{"name":"tag","in":"query","description":"OCI artifact tag","allowEmptyValue":true,"required":true,"schema":{"type":"string","description":"OCI artifact tag","example":"Sed nihil autem dolor blanditiis accusamus sit."},"example":"Neque sunt ut repellendus."},{"name":"repository","in":"path","description":"OCI artifact repository","required":true,"schema":{"type":"string","description":"OCI artifact repository","example":"Quaerat architecto."},"example":"Id et sit expedita."}],"responses":{"200":{"description":"OK response.","headers":{"Content-Disposition":{"description":"Content-Disposition header for downloading","schema":{"type":"string","description":"Content-Disposition header for downloading","example":"attachment; filename*=UTF-8''tidb-v7.5.0-darwin-arm64.tar.gz"},"example":"attachment; filename*=UTF-8''tidb-v7.5.0-darwin-arm64.tar.gz"},"Content-Length":{"description":"Length is the downloaded content length in bytes.","schema":{"type":"integer","description":"Length is the downloaded content length in bytes.","example":4194304,"format":"int64"},"example":4194304}},"content":{"application/octet-stream":{"schema":{"type":"string","format":"binary"}}}}}}},"/oci-files/{repository}":{"get":{"tags":["oci"],"summary":"list-files oci","operationId":"oci#list-files","parameters":[{"name":"tag","in":"query","description":"OCI artifact tag","allowEmptyValue":true,"required":true,"schema":{"type":"string","description":"OCI artifact tag","example":"Et qui quo."},"example":"Fugiat tempore sapiente dolorem et laborum impedit."},{"name":"repository","in":"path","description":"OCI artifact repository","required":true,"schema":{"type":"string","description":"OCI artifact repository","example":"Vero dolores amet possimus quasi libero nesciunt."},"example":"Debitis nostrum maiores est."}],"responses":{"200":{"description":"OK response.","content":{"application/json":{"schema":{"type":"array","items":{"type":"string","example":"Dolor itaque ullam."},"example":["Id dolorem dolor quasi qui.","Dolore et et fugit officia quia aperiam.","Quam et asperiores."]},"example":["Aut vero et.","Odit rerum at aperiam laboriosam odio.","Quam iusto assumenda."]}}}}}},"/s3-obj/{bucket}/{key}":{"get":{"tags":["ks3"],"summary":"download-object ks3","operationId":"ks3#download-object","parameters":[{"name":"bucket","in":"path","description":"bucket name","required":true,"schema":{"type":"string","description":"bucket name","example":"Ut nulla et itaque."},"example":"Molestiae dolorum."},{"name":"key","in":"path","description":"object key","required":true,"schema":{"type":"string","description":"object key","example":"Cupiditate recusandae qui."},"example":"Iste dolorem quasi eos."}],"responses":{"200":{"description":"OK response.","headers":{"Content-Disposition":{"description":"Content-Disposition header for downloading","schema":{"type":"string","description":"Content-Disposition header for downloading","example":"attachment; filename*=UTF-8''tidb-v7.5.0-darwin-arm64.tar.gz"},"example":"attachment; filename*=UTF-8''tidb-v7.5.0-darwin-arm64.tar.gz"},"Content-Length":{"description":"Length is the downloaded content length in bytes.","schema":{"type":"integer","description":"Length is the downloaded content length in bytes.","example":4194304,"format":"int64"},"example":4194304}},"content":{"application/octet-stream":{"schema":{"type":"string","format":"binary"}}}}}}}},"components":{},"tags":[{"name":"health","description":"Health service"},{"name":"oci","description":"OCI artifacts download service"},{"name":"ks3","description":"OCI artifacts download service"}]} \ No newline at end of file +{"openapi":"3.0.3","info":{"title":"Download OCI artifacts Service","description":"Service for downloading files from OCI artifact","version":"0.0.1"},"servers":[{"url":"http://localhost:8000"}],"paths":{"/healthz":{"get":{"tags":["health"],"summary":"healthz health","operationId":"health#healthz","responses":{"200":{"description":"OK response.","content":{"application/json":{"schema":{"type":"boolean","example":true},"example":true}}}}}},"/livez":{"get":{"tags":["health"],"summary":"livez health","operationId":"health#livez","responses":{"200":{"description":"OK response.","content":{"application/json":{"schema":{"type":"boolean","example":true},"example":true}}}}}},"/oci-file-sha256/{repository}":{"get":{"tags":["oci"],"summary":"download-file-sha256 oci","operationId":"oci#download-file-sha256","parameters":[{"name":"file","in":"query","description":"file name in OCI artifact","allowEmptyValue":true,"required":true,"schema":{"type":"string","description":"file name in OCI artifact","example":"Eligendi aut vero et neque odit."},"example":"At aperiam laboriosam odio ut quam."},{"name":"tag","in":"query","description":"OCI artifact tag","allowEmptyValue":true,"required":true,"schema":{"type":"string","description":"OCI artifact tag","example":"Assumenda aliquam est et aut."},"example":"Magnam a corporis."},{"name":"repository","in":"path","description":"OCI artifact repository","required":true,"schema":{"type":"string","description":"OCI artifact repository","example":"Sed nihil autem dolor blanditiis accusamus sit."},"example":"Neque sunt ut repellendus."}],"responses":{"200":{"description":"OK response.","headers":{"Content-Disposition":{"description":"Content-Disposition header for downloading","schema":{"type":"string","description":"Content-Disposition header for downloading","example":"attachment; filename*=UTF-8''tidb-v7.5.0-darwin-arm64.tar.gz.sha256"},"example":"attachment; filename*=UTF-8''tidb-v7.5.0-darwin-arm64.tar.gz.sha256"},"Content-Length":{"description":"Length is the downloaded content length in bytes.","schema":{"type":"integer","description":"Length is the downloaded content length in bytes.","example":4194304,"format":"int64"},"example":4194304}},"content":{"application/plain-text":{"schema":{"type":"string","format":"binary"}}}}}}},"/oci-file/{repository}":{"get":{"tags":["oci"],"summary":"download-file oci","operationId":"oci#download-file","parameters":[{"name":"tag","in":"query","description":"OCI artifact tag","allowEmptyValue":true,"required":true,"schema":{"type":"string","description":"OCI artifact tag","example":"v8.1.0_darwin_arm64"},"example":"v8.1.0_darwin_arm64"},{"name":"file","in":"query","description":"file name in OCI artifact","allowEmptyValue":true,"schema":{"type":"string","description":"file name in OCI artifact","example":"tidb-v7.5.0-darwin-arm64.tar.gz"},"example":"tidb-v7.5.0-darwin-arm64.tar.gz"},{"name":"file_regex","in":"query","description":"file name regex pattern in OCI artifact","allowEmptyValue":true,"schema":{"type":"string","description":"file name regex pattern in OCI artifact","example":"tidb-.+[.]tar[.]gz","format":"regexp"},"example":"tidb-.+[.]tar[.]gz"},{"name":"repository","in":"path","description":"OCI artifact repository","required":true,"schema":{"type":"string","description":"OCI artifact repository","example":"hub.pingcap.net/pingcap/tidb/package"},"example":"hub.pingcap.net/pingcap/tidb/package"}],"responses":{"200":{"description":"OK response.","headers":{"Content-Disposition":{"description":"Content-Disposition header for downloading","schema":{"type":"string","description":"Content-Disposition header for downloading","example":"attachment; filename*=UTF-8''tidb-v7.5.0-darwin-arm64.tar.gz"},"example":"attachment; filename*=UTF-8''tidb-v7.5.0-darwin-arm64.tar.gz"},"Content-Length":{"description":"Length is the downloaded content length in bytes.","schema":{"type":"integer","description":"Length is the downloaded content length in bytes.","example":4194304,"format":"int64"},"example":4194304}},"content":{"application/octet-stream":{"schema":{"type":"string","format":"binary"}}}}}}},"/oci-files/{repository}":{"get":{"tags":["oci"],"summary":"list-files oci","operationId":"oci#list-files","parameters":[{"name":"tag","in":"query","description":"OCI artifact tag","allowEmptyValue":true,"required":true,"schema":{"type":"string","description":"OCI artifact tag","example":"Dolor itaque ullam."},"example":"Repellendus id dolorem dolor quasi qui consectetur."},{"name":"repository","in":"path","description":"OCI artifact repository","required":true,"schema":{"type":"string","description":"OCI artifact repository","example":"Et et fugit."},"example":"Quia aperiam sunt quam et asperiores cupiditate."}],"responses":{"200":{"description":"OK response.","content":{"application/json":{"schema":{"type":"array","items":{"type":"string","example":"Qui itaque quos quaerat nulla."},"example":["Vel nobis asperiores at.","Dignissimos error et repellendus.","Nam quibusdam recusandae dolor voluptas."]},"example":["Et qui quo.","Fugiat tempore sapiente dolorem et laborum impedit.","Vero dolores amet possimus quasi libero nesciunt.","Debitis nostrum maiores est."]}}}}}},"/s3-obj/{bucket}/{key}":{"get":{"tags":["ks3"],"summary":"download-object ks3","operationId":"ks3#download-object","parameters":[{"name":"bucket","in":"path","description":"bucket name","required":true,"schema":{"type":"string","description":"bucket name","example":"Quaerat architecto."},"example":"Id et sit expedita."},{"name":"key","in":"path","description":"object key","required":true,"schema":{"type":"string","description":"object key","example":"Vero voluptatum perspiciatis omnis qui vel."},"example":"A at dolor soluta molestiae."}],"responses":{"200":{"description":"OK response.","headers":{"Content-Disposition":{"description":"Content-Disposition header for downloading","schema":{"type":"string","description":"Content-Disposition header for downloading","example":"attachment; filename*=UTF-8''tidb-v7.5.0-darwin-arm64.tar.gz"},"example":"attachment; filename*=UTF-8''tidb-v7.5.0-darwin-arm64.tar.gz"},"Content-Length":{"description":"Length is the downloaded content length in bytes.","schema":{"type":"integer","description":"Length is the downloaded content length in bytes.","example":4194304,"format":"int64"},"example":4194304}},"content":{"application/octet-stream":{"schema":{"type":"string","format":"binary"}}}}}}}},"components":{},"tags":[{"name":"health","description":"Health service"},{"name":"oci","description":"OCI artifacts download service"},{"name":"ks3","description":"OCI artifacts download service"}]} \ No newline at end of file diff --git a/dl/gen/http/openapi3.yaml b/dl/gen/http/openapi3.yaml index 5ee0f726..7102ea09 100644 --- a/dl/gen/http/openapi3.yaml +++ b/dl/gen/http/openapi3.yaml @@ -20,7 +20,7 @@ paths: schema: type: boolean example: true - example: false + example: true /livez: get: tags: @@ -35,7 +35,7 @@ paths: schema: type: boolean example: true - example: false + example: true /oci-file-sha256/{repository}: get: tags: @@ -51,8 +51,8 @@ paths: schema: type: string description: file name in OCI artifact - example: Vero voluptatum perspiciatis omnis qui vel. - example: A at dolor soluta molestiae. + example: Eligendi aut vero et neque odit. + example: At aperiam laboriosam odio ut quam. - name: tag in: query description: OCI artifact tag @@ -61,8 +61,8 @@ paths: schema: type: string description: OCI artifact tag - example: Sequi dolor voluptatem atque. - example: Nesciunt eum quia. + example: Assumenda aliquam est et aut. + example: Magnam a corporis. - name: repository in: path description: OCI artifact repository @@ -70,8 +70,8 @@ paths: schema: type: string description: OCI artifact repository - example: Nobis possimus eaque dolor sunt similique. - example: Quod nulla nostrum voluptatem soluta reprehenderit reiciendis. + example: Sed nihil autem dolor blanditiis accusamus sit. + example: Neque sunt ut repellendus. responses: "200": description: OK response. @@ -103,26 +103,35 @@ paths: summary: download-file oci operationId: oci#download-file parameters: + - name: tag + in: query + description: OCI artifact tag + allowEmptyValue: true + required: true + schema: + type: string + description: OCI artifact tag + example: v8.1.0_darwin_arm64 + example: v8.1.0_darwin_arm64 - name: file in: query description: file name in OCI artifact allowEmptyValue: true - required: true schema: type: string description: file name in OCI artifact - example: Est et. - example: Non magnam a corporis. - - name: tag + example: tidb-v7.5.0-darwin-arm64.tar.gz + example: tidb-v7.5.0-darwin-arm64.tar.gz + - name: file_regex in: query - description: OCI artifact tag + description: file name regex pattern in OCI artifact allowEmptyValue: true - required: true schema: type: string - description: OCI artifact tag - example: Sed nihil autem dolor blanditiis accusamus sit. - example: Neque sunt ut repellendus. + description: file name regex pattern in OCI artifact + example: tidb-.+[.]tar[.]gz + format: regexp + example: tidb-.+[.]tar[.]gz - name: repository in: path description: OCI artifact repository @@ -130,8 +139,8 @@ paths: schema: type: string description: OCI artifact repository - example: Quaerat architecto. - example: Id et sit expedita. + example: hub.pingcap.net/pingcap/tidb/package + example: hub.pingcap.net/pingcap/tidb/package responses: "200": description: OK response. @@ -171,8 +180,8 @@ paths: schema: type: string description: OCI artifact tag - example: Et qui quo. - example: Fugiat tempore sapiente dolorem et laborum impedit. + example: Dolor itaque ullam. + example: Repellendus id dolorem dolor quasi qui consectetur. - name: repository in: path description: OCI artifact repository @@ -180,8 +189,8 @@ paths: schema: type: string description: OCI artifact repository - example: Vero dolores amet possimus quasi libero nesciunt. - example: Debitis nostrum maiores est. + example: Et et fugit. + example: Quia aperiam sunt quam et asperiores cupiditate. responses: "200": description: OK response. @@ -191,15 +200,16 @@ paths: type: array items: type: string - example: Dolor itaque ullam. + example: Qui itaque quos quaerat nulla. example: - - Id dolorem dolor quasi qui. - - Dolore et et fugit officia quia aperiam. - - Quam et asperiores. + - Vel nobis asperiores at. + - Dignissimos error et repellendus. + - Nam quibusdam recusandae dolor voluptas. example: - - Aut vero et. - - Odit rerum at aperiam laboriosam odio. - - Quam iusto assumenda. + - Et qui quo. + - Fugiat tempore sapiente dolorem et laborum impedit. + - Vero dolores amet possimus quasi libero nesciunt. + - Debitis nostrum maiores est. /s3-obj/{bucket}/{key}: get: tags: @@ -214,8 +224,8 @@ paths: schema: type: string description: bucket name - example: Ut nulla et itaque. - example: Molestiae dolorum. + example: Quaerat architecto. + example: Id et sit expedita. - name: key in: path description: object key @@ -223,8 +233,8 @@ paths: schema: type: string description: object key - example: Cupiditate recusandae qui. - example: Iste dolorem quasi eos. + example: Vero voluptatum perspiciatis omnis qui vel. + example: A at dolor soluta molestiae. responses: "200": description: OK response. diff --git a/dl/gen/oci/service.go b/dl/gen/oci/service.go index 47b899b5..41eac030 100644 --- a/dl/gen/oci/service.go +++ b/dl/gen/oci/service.go @@ -48,7 +48,9 @@ type DownloadFilePayload struct { // OCI artifact tag Tag string // file name in OCI artifact - File string + File *string + // file name regexp pattern + FileRegex *string } // DownloadFileResult is the result type of the oci service download-file diff --git a/dl/go.mod b/dl/go.mod index b6175a17..104b2585 100644 --- a/dl/go.mod +++ b/dl/go.mod @@ -1,30 +1,30 @@ module github.com/PingCAP-QE/ee-apps/dl -go 1.21 +go 1.23 require ( github.com/ks3sdklib/aws-sdk-go v1.3.0 github.com/opencontainers/image-spec v1.1.0 - goa.design/goa/v3 v3.16.1 + goa.design/goa/v3 v3.19.0 gopkg.in/yaml.v3 v3.0.1 oras.land/oras-go/v2 v2.5.0 ) require ( - github.com/AnatolyRugalev/goregen v0.1.0 // indirect + github.com/davecgh/go-spew v1.1.1 // indirect github.com/dimfeld/httppath v0.0.0-20170720192232-ee938bf73598 // indirect - github.com/go-chi/chi/v5 v5.0.12 // indirect + github.com/go-chi/chi/v5 v5.1.0 // indirect github.com/google/uuid v1.6.0 // indirect - github.com/gorilla/websocket v1.5.1 // indirect + github.com/gorilla/websocket v1.5.3 // indirect github.com/kr/pretty v0.3.1 // indirect github.com/manveru/faker v0.0.0-20171103152722-9fbc68a78c4d // indirect github.com/opencontainers/go-digest v1.0.0 // indirect - github.com/rogpeppe/go-internal v1.11.0 // indirect - github.com/sergi/go-diff v1.3.1 // indirect - golang.org/x/mod v0.17.0 // indirect - golang.org/x/net v0.24.0 // indirect - golang.org/x/sync v0.7.0 // indirect - golang.org/x/text v0.14.0 // indirect - golang.org/x/tools v0.20.0 // indirect + github.com/pmezard/go-difflib v1.0.0 // indirect + github.com/rogpeppe/go-internal v1.12.0 // indirect + github.com/stretchr/testify v1.9.0 // indirect + golang.org/x/mod v0.21.0 // indirect + golang.org/x/sync v0.8.0 // indirect + golang.org/x/text v0.18.0 // indirect + golang.org/x/tools v0.25.0 // indirect gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c // indirect ) diff --git a/dl/go.sum b/dl/go.sum index 93468a56..489b755c 100644 --- a/dl/go.sum +++ b/dl/go.sum @@ -1,18 +1,15 @@ -github.com/AnatolyRugalev/goregen v0.1.0 h1:xrdXkLaskMnbxW0x4FWNj2yoednv0X2bcTBWpuJGYfE= -github.com/AnatolyRugalev/goregen v0.1.0/go.mod h1:sVlY1tjcirqLBRZnCcIq1+7/Lwmqz5g7IK8AStjOVzI= github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/dimfeld/httppath v0.0.0-20170720192232-ee938bf73598 h1:MGKhKyiYrvMDZsmLR/+RGffQSXwEkXgfLSA08qDn9AI= github.com/dimfeld/httppath v0.0.0-20170720192232-ee938bf73598/go.mod h1:0FpDmbrt36utu8jEmeU05dPC9AB5tsLYVVi+ZHfyuwI= -github.com/go-chi/chi/v5 v5.0.12 h1:9euLV5sTrTNTRUU9POmDUvfxyj6LAABLUcEWO+JJb4s= -github.com/go-chi/chi/v5 v5.0.12/go.mod h1:DslCQbL2OYiznFReuXYUmQ2hGd1aDpCnlMNITLSKoi8= +github.com/go-chi/chi/v5 v5.1.0 h1:acVI1TYaD+hhedDJ3r54HyA6sExp3HfXq7QWEEY/xMw= +github.com/go-chi/chi/v5 v5.1.0/go.mod h1:DslCQbL2OYiznFReuXYUmQ2hGd1aDpCnlMNITLSKoi8= github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= -github.com/gorilla/websocket v1.5.1 h1:gmztn0JnHVt9JZquRuzLw3g4wouNVzKL15iLr/zn/QY= -github.com/gorilla/websocket v1.5.1/go.mod h1:x3kM2JMyaluk02fnUJpQuwD2dCS5NDG2ZHL0uE0tcaY= -github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= +github.com/gorilla/websocket v1.5.3 h1:saDtZ6Pbx/0u+bgYQ3q96pZgCzfhKXGPqt7kZ72aNNg= +github.com/gorilla/websocket v1.5.3/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= @@ -34,37 +31,29 @@ github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e/go.mod h1:pJLUxLENpZxwdsK github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/rogpeppe/go-internal v1.9.0/go.mod h1:WtVeX8xhTBvf0smdhujwtBcq4Qrzq/fJaraNFVN+nFs= -github.com/rogpeppe/go-internal v1.11.0 h1:cWPaGQEPrBb5/AsnsZesgZZ9yb1OQ+GOISoDNXVBh4M= -github.com/rogpeppe/go-internal v1.11.0/go.mod h1:ddIwULY96R17DhadqLgMfk9H9tvdUzkipdSkR5nkCZA= -github.com/sergi/go-diff v1.3.1 h1:xkr+Oxo4BOQKmkn/B9eMK0g5Kg/983T9DqqPHwYqD+8= -github.com/sergi/go-diff v1.3.1/go.mod h1:aMJSSKb2lpPvRNec0+w3fl7LP9IOFzdc9Pa4NFbPK1I= +github.com/rogpeppe/go-internal v1.12.0 h1:exVL4IDcn6na9z1rAb56Vxr+CgyK3nn3O+epU5NdKM8= +github.com/rogpeppe/go-internal v1.12.0/go.mod h1:E+RYuTGaKKdloAfM02xzb0FW3Paa99yedzYV+kq4uf4= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= -github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg= github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= -goa.design/goa/v3 v3.16.1 h1:yZwbKrfMpE8+sz0uf+n+BtArVOFQ0kNSC0twQKwQb04= -goa.design/goa/v3 v3.16.1/go.mod h1:Yd42LR0PYDbHSbsbF3vNd4YY/O+LG20Jb7+IyNdkQic= -golang.org/x/mod v0.17.0 h1:zY54UmvipHiNd+pm+m0x9KhZ9hl1/7QNMyxXbc6ICqA= -golang.org/x/mod v0.17.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= -golang.org/x/net v0.24.0 h1:1PcaxkF854Fu3+lvBIx5SYn9wRlBzzcnHZSiaFFAb0w= -golang.org/x/net v0.24.0/go.mod h1:2Q7sJY5mzlzWjKtYUEXSlBWCdyaioyXzRB2RtU8KVE8= -golang.org/x/sync v0.7.0 h1:YsImfSBoP9QPYL0xyKJPq0gcaJdG3rInoqxTWbfQu9M= -golang.org/x/sync v0.7.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= -golang.org/x/text v0.14.0 h1:ScX5w1eTa3QqT8oi6+ziP7dTV1S2+ALU0bI+0zXKWiQ= -golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= -golang.org/x/tools v0.20.0 h1:hz/CVckiOxybQvFw6h7b/q80NTr9IUQb4s1IIzW7KNY= -golang.org/x/tools v0.20.0/go.mod h1:WvitBU7JJf6A4jOdg4S1tviW9bhUxkgeCui/0JHctQg= +goa.design/goa/v3 v3.19.0 h1:veBy7uz0dzR7v8bVkV7Mcwd5si3gTVoxHUcXU1534aA= +goa.design/goa/v3 v3.19.0/go.mod h1:x6hUwtv/+E/sUzoQlCkrjXiB7ZZOgrx8r8h8rHKb6tQ= +golang.org/x/mod v0.21.0 h1:vvrHzRwRfVKSiLrG+d4FMl/Qi4ukBCE6kZlTUkDYRT0= +golang.org/x/mod v0.21.0/go.mod h1:6SkKJ3Xj0I0BrPOZoBy3bdMptDDU9oJrpohJ3eWZ1fY= +golang.org/x/sync v0.8.0 h1:3NFvSEYkUoMifnESzZl15y791HH1qU2xm6eCJU5ZPXQ= +golang.org/x/sync v0.8.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= +golang.org/x/text v0.18.0 h1:XvMDiNzPAl0jr17s6W9lcaIhGUfUORdGCNsuLmPG224= +golang.org/x/text v0.18.0/go.mod h1:BuEKDfySbSR4drPmRPG/7iBdf8hvFMuRexcpahXilzY= +golang.org/x/tools v0.25.0 h1:oFU9pkj/iJgs+0DT+VMHrx+oBKs/LJMV+Uvg78sl+fE= +golang.org/x/tools v0.25.0/go.mod h1:/vtpO8WL1N9cQC3FN5zPqb//fRXskFHbLKk4OW1Q7rg= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= -gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= -gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= -gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/dl/oci.go b/dl/oci.go index 209ecf39..d23d6b79 100644 --- a/dl/oci.go +++ b/dl/oci.go @@ -2,10 +2,13 @@ package dl import ( "context" + "errors" + "fmt" "io" "log" "net/url" "os" + "regexp" "strings" oci "github.com/PingCAP-QE/ee-apps/dl/gen/oci" @@ -79,20 +82,49 @@ func (s *ocisrvc) getTargetRepo(repo string) (*remote.Repository, error) { // DownloadFile implements download-file. func (s *ocisrvc) DownloadFile(ctx context.Context, p *oci.DownloadFilePayload) (res *oci.DownloadFileResult, resp io.ReadCloser, err error) { - s.logger.Print("oci.download-files") + s.logger.Print("oci.download-file") repository, err := s.getTargetRepo(p.Repository) if err != nil { return nil, nil, err } - rc, length, err := pkgoci.NewFileReadCloser(ctx, repository, p.Tag, p.File) + + if p.File != nil { + return s.downloadFile(ctx, repository, p.Tag, *p.File) + } + + if p.FileRegex != nil { + pattern, err := regexp.Compile(*p.FileRegex) + if err != nil { + return nil, nil, oci.MakeInvalidFilePath(err) + } + + files, err := pkgoci.ListFiles(ctx, repository, p.Tag) + if err != nil { + return nil, nil, oci.MakeInvalidFilePath(err) + } + + for _, file := range files { + if pattern.MatchString(file) { + return s.downloadFile(ctx, repository, p.Tag, file) + } + } + + return nil, nil, oci.MakeInvalidFilePath(fmt.Errorf("could not locate file for download: %s", *p.FileRegex)) + } + + return nil, nil, oci.MakeInvalidFilePath(errors.New("none `file` or `file_regex` param given")) +} + +func (s *ocisrvc) downloadFile(ctx context.Context, repo *remote.Repository, tag, file string) (res *oci.DownloadFileResult, resp io.ReadCloser, err error) { + rc, length, err := pkgoci.NewFileReadCloser(ctx, repo, tag, file) if err != nil { return nil, nil, err } res = &oci.DownloadFileResult{ Length: length, - ContentDisposition: `attachment; filename*=UTF-8''` + url.QueryEscape(p.File), + ContentDisposition: `attachment; filename*=UTF-8''` + url.QueryEscape(file), } return res, rc, nil }