package api import ( "bytes" "context" "encoding/json" "errors" "fmt" "io" "maps" "net/http" "net/url" "os" "regexp" "slices" "sort" "strconv" "strings" "time" "github.com/MakeNowJust/heredoc/v2" "github.com/spf13/cobra" jsonPretty "github.com/tidwall/pretty" "gitlab.com/gitlab-org/cli/internal/api" "gitlab.com/gitlab-org/cli/internal/cmdutils" "gitlab.com/gitlab-org/cli/internal/glinstance" "gitlab.com/gitlab-org/cli/internal/glrepo" "gitlab.com/gitlab-org/cli/internal/iostreams" "gitlab.com/gitlab-org/cli/internal/mcpannotations" ) type options struct { io *iostreams.IOStreams apiClient func(repoHost string) (*api.Client, error) baseRepo func() (glrepo.Interface, error) branch func() (string, error) hostname string requestMethod string requestMethodPassed bool requestPath string requestInputFile string fields []fieldFlag formFields []string requestHeaders []string showResponseHeaders bool paginate bool silent bool outputFormat string placeholderFlags []string placeholders map[string]string placeholderRE *regexp.Regexp } type fieldFlag struct { spec string raw bool } // fieldFlagList is the one ordered list --field and --raw-field collect into. // pflag keeps a slice per flag, losing the interleave order, which is part of // the request: repeated names ending in [] send every value as typed. changed // is pflag's reset bit, shared so neither flag discards the other's values. type fieldFlagList struct { fields *[]fieldFlag changed bool } type fieldFlagValue struct { list *fieldFlagList raw bool } // Type keeps pflag's string-array name, so --help and the docs read as before. func (v *fieldFlagValue) Type() string { return "stringArray" } func (v *fieldFlagValue) Set(spec string) error { if !v.list.changed { *v.list.fields = nil v.list.changed = true } *v.list.fields = append(*v.list.fields, fieldFlag{spec: spec, raw: v.raw}) return nil } func (v *fieldFlagValue) String() string { var specs []string for _, f := range *v.list.fields { if f.raw == v.raw { specs = append(specs, f.spec) } } if len(specs) == 0 { return "" } return "[" + strings.Join(specs, ",") + "]" } func NewCmdApi(f cmdutils.Factory, runF func(*options) error) *cobra.Command { opts := options{ io: f.IO(), apiClient: f.ApiClient, baseRepo: f.BaseRepo, branch: f.Branch, } cmd := &cobra.Command{ Use: "api ", Short: "Make an authenticated request to the GitLab API.", Long: heredoc.Docf(` Makes an authenticated HTTP request to the GitLab API, and prints the response. Specify either a path to a GitLab API v4 endpoint, or %[1]sgraphql%[1]s to access the GitLab GraphQL API. For more information, see: - [GitLab REST API documentation](https://docs.gitlab.com/api/) - [GitLab GraphQL documentation](https://docs.gitlab.com/api/graphql/) If the current directory is a Git directory, this command uses the GitLab authenticated host in the current directory. Otherwise, %[1]sgitlab.com%[1]s is used. To override the GitLab hostname, use %[1]s--hostname%[1]s. When used in the endpoint argument, these placeholder values are replaced with values from the repository of the current directory: - %[1]s:branch%[1]s - %[1]s:fullpath%[1]s - %[1]s:group%[1]s - %[1]s:id%[1]s - %[1]s:namespace%[1]s - %[1]s:repo%[1]s - %[1]s:user%[1]s - %[1]s:username%[1]s To define your own placeholder, pass %[1]s--placeholder =%[1]s and use %[1]s:%[1]s in the endpoint or in %[1]s--field%[1]s values. In the endpoint, the value is encoded as a single path segment. In %[1]s--field%[1]s values, it is sent unchanged. To put a value in a query string, pass it as a field instead: %[1]s-X GET -F =:%[1]s. A custom placeholder cannot reuse the name of a built-in placeholder. The default HTTP request method is %[1]sGET%[1]s when no parameters are added, and %[1]sPOST%[1]s otherwise. To override the method, use %[1]s--method%[1]s. Pass one or more %[1]s--raw-field%[1]s values in %[1]skey=value%[1]s format to add JSON-encoded string parameters to the %[1]sPOST%[1]s body. The %[1]s--field%[1]s flag behaves like %[1]s--raw-field%[1]s but converts values based on their format: - Literal values %[1]strue%[1]s, %[1]sfalse%[1]s, %[1]snull%[1]s, and integer numbers are converted to the matching JSON types. - Values starting with %[1]s[%[1]s or %[1]s{%[1]s are parsed as JSON arrays or objects (e.g. %[1]s-F 'topics=["my-topic","GitLab"]'%[1]s). Invalid JSON returns an error. The value must start with the bracket or brace: a leading space, as in %[1]s-F 'topics= ["a"]'%[1]s, is part of the value, so it is sent as a string. - Placeholder values %[1]s:namespace%[1]s, %[1]s:repo%[1]s, and %[1]s:branch%[1]s are populated with values from the repository of the current directory, including inside JSON arrays and objects (e.g. %[1]s-F 'input={"projectPath":":fullpath"}'%[1]s). - If the value starts with %[1]s@%[1]s, the rest of the value is interpreted as a filename to read the value from. Pass %[1]s-%[1]s to read from standard input. %[1]s--raw-field%[1]s does not parse JSON arrays or objects; those values are sent as strings. To pass a JSON body literally, use %[1]s--input%[1]s. A bracketed %[1]s--raw-field%[1]s value such as %[1]s-f 'scopes=[api,read_api]'%[1]s is sent as the literal string %[1]s"[api,read_api]"%[1]s. Earlier versions converted that shape into an array on request bodies, which %[1]s--raw-field%[1]s never documented. Use %[1]s-F 'scopes=["api","read_api"]'%[1]s for an array. Field names are never parsed. A bracketed name is only meaningful where the fields become URL query parameters. That means %[1]sGET%[1]s and %[1]sDELETE%[1]s requests, and any request where %[1]s--input%[1]s supplies the body. There the name is used literally and percent-encoded, so %[1]s-f 'position[base_sha]=abc'%[1]s arrives as the nested parameter it describes. A name ending in %[1]s[]%[1]s collects values instead of holding one, which is how the API reads an array. Repeating the name sends every value, in the order the flags were given: %[1]s-X GET -f 'ids[]=1' -f 'ids[]=2'%[1]s sends both. Because every value is sent, %[1]s--field%[1]s does not override %[1]s--raw-field%[1]s of the same name, as it does for other names. Any other name holds a single value. Among values from the same flag, the last one wins. When both flags name it, the %[1]s--field%[1]s value wins, whichever order the two were given in. The %[1]s-F 'ids=[1,2]'%[1]s spelling emits %[1]sids[]%[1]s as well. Spelling one query parameter both ways is therefore an error. Passing %[1]s-f 'ids[]=1'%[1]s with %[1]s-F 'ids=[2,3]'%[1]s is rejected, because nothing says which order the three values belong in. In a JSON request body a field name containing a bracket is an error. Such a name would be sent as a single literal key, which the API ignores. The request would succeed without doing what the name asked. Pass the value as JSON instead, as in %[1]s-F 'position={"base_sha":"abc"}'%[1]s or %[1]s-F 'ids=[1,2]'%[1]s. Or use %[1]s--input%[1]s. With no %[1]s--method%[1]s, adding any field makes the request a %[1]sPOST%[1]s. Sending bracketed names as query parameters therefore needs an explicit %[1]s--method GET%[1]s. %[1]s--form%[1]s is unaffected: its part names are always literal. For GraphQL requests, all fields other than %[1]squery%[1]s and %[1]soperationName%[1]s are interpreted as GraphQL variables. The bracket rule applies to them unchanged: where the request sends a JSON body, a bracketed variable name is an error. To send data as %[1]smultipart/form-data%[1]s instead of JSON, use %[1]s--form%[1]s. This is required for API endpoints that accept file uploads, such as wiki attachments. Pass one or more %[1]s--form%[1]s values in %[1]skey=value%[1]s format. To upload a file, prefix the value with %[1]s@%[1]s followed by the file path. Pass %[1]s-%[1]s to read from standard input. Do not combine %[1]s--form%[1]s with %[1]s--field%[1]s, %[1]s--raw-field%[1]s, or %[1]s--input%[1]s. To pass a raw request body, use %[1]s--input%[1]s with a file path. Pass %[1]s-%[1]s to read from standard input. In this mode, parameters specified with %[1]s--field%[1]s flags are serialized into URL query parameters. A %[1]s--input%[1]s body is sent as %[1]sapplication/json%[1]s, which is what most GitLab endpoints require to parse it. To send a body of another type, set the header yourself with %[1]s--header 'Content-Type: ...'%[1]s; an explicit header is never overridden. In %[1]s--paginate%[1]s mode, all pages of results are requested sequentially until no more pages of results remain. For GraphQL requests: - The original query must accept an %[1]s$endCursor: String%[1]s variable. - The query must fetch the %[1]spageInfo{ hasNextPage, endCursor }%[1]s set of fields from a collection. %[1]sglab api%[1]s does not retry a failed request. When GitLab answers with HTTP 429, %[1]sglab api%[1]s exits with an error that says how long to wait before you retry, taken from the %[1]sRetry-After%[1]s or %[1]sRateLimit-Reset%[1]s response header. The %[1]s--output%[1]s flag controls the output format: - %[1]sjson%[1]s (default): Pretty-printed JSON. Arrays are output as a single JSON array. - %[1]sndjson%[1]s: Newline-delimited JSON, also known as JSON Lines. Each array element or object is output on a separate line. This format is more memory-efficient for large datasets and works well with tools like %[1]sjq%[1]s. For ndjson format specifications, see the [ndjson spec](https://github.com/ndjson/ndjson-spec) and [JSON Lines](https://jsonlines.org/). `, "`"), Example: heredoc.Doc(` # List releases for the current project, expanding the :fullpath placeholder glab api projects/:fullpath/releases # List issues for a project by URL-encoded path glab api projects/gitlab-com%2Fwww-gitlab-com/issues # Get a branch whose name contains a slash, using a custom placeholder glab api projects/:fullpath/repository/branches/:target --placeholder target=feature/login # Upload a file to a project wiki glab api --method POST projects/:fullpath/wikis/attachments --form "file=@./image.png" --form "branch=main" # Debug the HTTP request and response, including headers and body. # Use --input to send a complete JSON request body from stdin. echo '{"allowed_to_push":[{"user_id":1}]}' | GLAB_DEBUG_HTTP=1 glab api -X PATCH "projects/:fullpath/protected_branches/main" --input - # Fetch all pages of issues glab api issues --paginate # Fetch all pages of issues as newline-delimited JSON glab api issues --paginate --output ndjson # Pipe paginated output to jq to filter open issues glab api issues --paginate --output ndjson | jq 'select(.state == "opened")' # Run a simple GraphQL query glab api graphql -f query="query { currentUser { username } }" # Run a multi-line GraphQL query for project metadata glab api graphql -f query=' query { project(fullPath: "gitlab-org/gitlab-docs") { name forksCount statistics { wikiSize } issuesEnabled boards { nodes { id name } } } } ' # Run a paginated GraphQL query using an endCursor variable glab api graphql --paginate -f query=' query($endCursor: String) { project(fullPath: "gitlab-org/graphql-sandbox") { name issues(first: 2, after: $endCursor) { edges { node { title } } pageInfo { endCursor hasNextPage } } } }' `), Annotations: map[string]string{ mcpannotations.Destructive: "true", "help:environment": heredoc.Doc(` GITLAB_TOKEN, OAUTH_TOKEN (in order of precedence): an authentication token for API requests. GITLAB_HOST, GITLAB_URI, GITLAB_URL: specify a GitLab host to make request to. GLAB_DEBUG_HTTP: print the full HTTP request and response, including headers and body, to stderr. `), }, Args: cobra.ExactArgs(1), RunE: func(cmd *cobra.Command, args []string) error { opts.complete(cmd, args) if err := opts.validate(cmd); err != nil { return err } if runF != nil { return runF(&opts) } return opts.run(cmd.Context()) }, } fl := cmd.Flags() fl.StringVar(&opts.hostname, "hostname", "", "The GitLab hostname for the request. Defaults to gitlab.com, or the authenticated host in the current Git directory.") fl.StringVarP(&opts.requestMethod, "method", "X", "GET", "The HTTP method for the request.") fieldList := &fieldFlagList{fields: &opts.fields} fl.VarP(&fieldFlagValue{list: fieldList}, "field", "F", "Add a parameter of inferred type. Using this flag changes the default HTTP method to POST.") fl.VarP(&fieldFlagValue{list: fieldList, raw: true}, "raw-field", "f", "Add a string parameter.") fl.StringArrayVar(&opts.formFields, "form", nil, "Add a multipart form field. To upload a file, prefix the value with @ followed by the file path. To read from standard input, use @- (at most once). Using this flag changes the default HTTP method to POST.") fl.StringArrayVarP(&opts.requestHeaders, "header", "H", nil, "Add an additional HTTP request header.") fl.BoolVarP(&opts.showResponseHeaders, "include", "i", false, "Include HTTP response headers in the output.") fl.BoolVar(&opts.paginate, "paginate", false, "Make additional HTTP requests to fetch all pages of results.") fl.StringVar(&opts.requestInputFile, "input", "", "The file to use as the body for the HTTP request. Sent as 'application/json' unless you set another 'Content-Type' with --header.") fl.BoolVar(&opts.silent, "silent", false, "Do not print the response body.") fl.Var(cmdutils.NewEnumValue([]string{"json", "ndjson"}, "json", &opts.outputFormat), "output", "Format output as: json, ndjson.") fl.StringArrayVar(&opts.placeholderFlags, "placeholder", nil, "Define a custom placeholder in = format, expanded from :. Repeat the flag to define more than one.") cmd.MarkFlagsMutuallyExclusive("paginate", "input") cmd.MarkFlagsMutuallyExclusive("form", "field") cmd.MarkFlagsMutuallyExclusive("form", "raw-field") cmd.MarkFlagsMutuallyExclusive("form", "input") return cmd } func (o *options) complete(cmd *cobra.Command, args []string) { o.requestPath = args[0] o.requestMethodPassed = cmd.Flags().Changed("method") } func (o *options) validate(cmd *cobra.Command) error { if cmd.Flags().Changed("hostname") { if err := glinstance.HostnameValidator(o.hostname); err != nil { return &cmdutils.FlagError{Err: fmt.Errorf("error parsing --hostname: %w", err)} } } if o.paginate && !strings.EqualFold(o.requestMethod, http.MethodGet) && o.requestPath != "graphql" { return &cmdutils.FlagError{Err: errors.New("the '--paginate' option is not supported for non-GET requests")} } if o.outputFormat != "json" && o.outputFormat != "ndjson" { return &cmdutils.FlagError{Err: fmt.Errorf("invalid output format %q: must be 'json' or 'ndjson'", o.outputFormat)} } stdinCount := 0 for _, f := range o.formFields { if strings.HasSuffix(f, "=@-") { stdinCount++ } } if stdinCount > 1 { return &cmdutils.FlagError{Err: errors.New("'@-' (stdin) can only be used once across all --form fields")} } return o.parsePlaceholders() } func (o *options) parsePlaceholders() error { if len(o.placeholderFlags) == 0 { return nil } o.placeholders = make(map[string]string, len(o.placeholderFlags)) for _, spec := range o.placeholderFlags { name, value, ok := strings.Cut(spec, "=") switch { case !ok: return &cmdutils.FlagError{Err: fmt.Errorf("invalid --placeholder %q: expected =", spec)} case !placeholderNameRE.MatchString(name): return &cmdutils.FlagError{Err: fmt.Errorf("invalid --placeholder name %q: must start with a letter, contain only letters, digits, '_', or '-', and not end with '-'", name)} case slices.Contains(Placeholders, name): return &cmdutils.FlagError{Err: fmt.Errorf("invalid --placeholder name %q: conflicts with a built-in placeholder", name)} } if _, dup := o.placeholders[name]; dup { return &cmdutils.FlagError{Err: fmt.Errorf("invalid --placeholder name %q: defined more than once", name)} } o.placeholders[name] = value } names := slices.Concat(Placeholders, slices.Collect(maps.Keys(o.placeholders))) slices.SortStableFunc(names, func(a, b string) int { return len(b) - len(a) }) o.placeholderRE = regexp.MustCompile(`:(` + strings.Join(names, "|") + `)\b`) return nil } // EffectiveMethod reports the HTTP method an invocation of this command uses. func EffectiveMethod(cmd *cobra.Command) string { fl := cmd.Flags() bodyFlags := []string{"field", "raw-field", "form", "input"} method := resolveMethod( fl.Lookup("method").Value.String(), fl.Changed("method"), slices.ContainsFunc(bodyFlags, fl.Changed), ) if method == "" { return http.MethodGet } return strings.ToUpper(method) } // resolveMethod applies the documented default: the request is a GET unless // parameters, a form, or an input file are supplied, and an explicit --method // always wins. Shared with the telemetry hook so the two cannot drift. func resolveMethod(method string, methodPassed, hasBody bool) string { if !methodPassed && hasBody { return http.MethodPost } return method } func (o *options) run(ctx context.Context) error { // Parsing needs the method: it decides whether a bracketed name is an error. method := o.methodForRequest() params, rawKeys, err := parseFields(o, o.fieldsBecomeJSONBody(method)) if err != nil { return err } isGraphQL := o.requestPath == "graphql" requestPath, err := fillPlaceholders(o.requestPath, o, true) if err != nil { return fmt.Errorf("unable to expand placeholder in path: %w", err) } requestHeaders := o.requestHeaders var requestBody any = params var bodyIsJSON bool o.warnOnLegacyRawArrays(method, params, rawKeys) if o.paginate && !isGraphQL { requestPath = addPerPage(requestPath, 100, params) } if o.requestInputFile != "" { file, size, err := openUserFile(o.requestInputFile, o.io.In) if err != nil { return err } defer file.Close() requestPath, err = parseQuery(requestPath, params) if err != nil { return err } requestBody = file // GitLab rejects a body with no Content-Type with HTTP 415. bodyIsJSON = true if size >= 0 { requestHeaders = append([]string{fmt.Sprintf("Content-Length: %d", size)}, requestHeaders...) } } else if len(o.formFields) > 0 { body, contentType := buildMultipartBody(o.formFields, o.io.In) requestBody = body requestHeaders = append([]string{fmt.Sprintf("Content-Type: %s", contentType)}, requestHeaders...) } headersOutputStream := o.io.StdOut if o.silent { o.io.StdOut = io.Discard } else { err := o.io.StartPager() if err != nil { return err } defer o.io.StopPager() } var repoHost string if o.hostname != "" { repoHost = o.hostname } else { // NOTE: check if repository is available br, err := o.baseRepo() if err == nil { repoHost = br.RepoHost() } } client, err := o.apiClient(repoHost) if err != nil { return err } hasNextPage := true for hasNextPage { resp, err := httpRequest(ctx, client, method, requestPath, requestBody, requestHeaders, bodyIsJSON) if err != nil { return err } endCursor, err := processResponse(resp, o, headersOutputStream) _ = resp.Body.Close() if err != nil { return err } if !o.paginate { break } if isGraphQL { hasNextPage = endCursor != "" if hasNextPage { params["endCursor"] = endCursor } } else { requestPath, hasNextPage = findNextPage(resp) // The Link URL already carries the previous request's parameters, so // rebuilding the query duplicates them and can override page. Only a // query method puts fields in the query; any other method carries // them in the body and has to keep sending it. if isQueryMethod(method) { requestBody = nil } } if hasNextPage && o.showResponseHeaders { o.io.LogInfo() } } return nil } func processResponse(resp *http.Response, opts *options, headersOutputStream io.Writer) (string, error) { if opts.showResponseHeaders { fmt.Fprintln(headersOutputStream, resp.Proto, resp.Status) //nolint:forbidigo // headersOutputStream is a generic io.Writer, passed io.Discard in tests printHeaders(headersOutputStream, resp.Header, opts.io.ColorEnabled()) fmt.Fprint(headersOutputStream, "\r\n") //nolint:forbidigo // headersOutputStream is a generic io.Writer, passed io.Discard in tests } if resp.StatusCode == http.StatusNoContent { return "", nil } var responseBody io.Reader = resp.Body isJSON, _ := regexp.MatchString(`[/+]json(;|$)`, resp.Header.Get("Content-Type")) var serverError string if isJSON && (opts.requestPath == "graphql" || resp.StatusCode >= http.StatusBadRequest) { var err error responseBody, serverError, err = parseErrorResponse(responseBody, resp.StatusCode) if err != nil { return "", err } } var bodyCopy *bytes.Buffer isGraphQLPaginate := isJSON && resp.StatusCode == http.StatusOK && opts.paginate && opts.requestPath == "graphql" if isGraphQLPaginate { bodyCopy = &bytes.Buffer{} responseBody = io.TeeReader(responseBody, bodyCopy) } var err error switch { case opts.outputFormat == "ndjson" && isJSON && resp.StatusCode == http.StatusOK: err = streamNDJSON(responseBody, opts.io.StdOut) case isJSON && opts.io.ColorEnabled(): out := &bytes.Buffer{} _, err = io.Copy(out, responseBody) if err == nil { result := jsonPretty.Color(jsonPretty.Pretty(out.Bytes()), nil) _, err = fmt.Fprintln(opts.io.StdOut, string(result)) //nolint:forbidigo // write error must propagate to the caller } default: _, err = io.Copy(opts.io.StdOut, responseBody) } if err != nil { return "", err } if serverError != "" || resp.StatusCode > 299 { if serverError != "" { opts.io.LogErrorf("glab: %s\n", serverError) } else { opts.io.LogErrorf("glab: HTTP %d\n", resp.StatusCode) } if hint := api.RateLimitHint(resp, time.Now()); hint != "" { opts.io.LogErrorf("glab: %s\n", hint) } return "", cmdutils.SilentError } if isGraphQLPaginate { return findEndCursor(bodyCopy), nil } return "", nil } // streamNDJSON streams JSON response as newline-delimited JSON. // If the response is a JSON array, each element is written as a separate line. // If the response is a single JSON object, it's written as-is with a newline. func streamNDJSON(body io.Reader, out io.Writer) error { dec := json.NewDecoder(body) dec.UseNumber() // Peek at the first token to determine if it's an array or object token, err := dec.Token() if err != nil { if errors.Is(err, io.EOF) { return nil } return err } // Check if it's an array if delim, ok := token.(json.Delim); ok && delim == '[' { // Stream each array element as a separate line for dec.More() { var element json.RawMessage if err := dec.Decode(&element); err != nil { return err } if _, err := fmt.Fprintln(out, string(element)); err != nil { //nolint:forbidigo // out is a generic io.Writer parameter for streaming, not IOStreams directly return err } } // Consume the closing bracket if _, err := dec.Token(); err != nil { return err } return nil } // For non-array JSON, simply copy the entire body and add newline // Reset the body reader since we consumed the first token var tokenStr string switch v := token.(type) { case json.Delim: tokenStr = string(rune(v)) default: // For primitive values (string, number, bool, null), marshal back to JSON b, _ := json.Marshal(v) //nolint:forbidigo // re-encoding a parsed JSON token value into a string, not writing to stdout tokenStr = string(b) } fullBody := io.MultiReader(strings.NewReader(tokenStr), dec.Buffered(), body) // Copy directly to output with newline if _, err := io.Copy(out, fullBody); err != nil { return err } _, err = out.Write([]byte("\n")) return err } // Placeholders are the tokens `glab api` expands in a request path or field // value, without their leading colon. Exported so documentation and the bundled // skills can be checked against the real set rather than a copy of it: a token // that is neither here nor defined with --placeholder is sent to the API verbatim. // // Order matters. The regexp alternation below tries these in sequence, so a // longer token must precede any token that prefixes it. var Placeholders = []string{ "group/:namespace/:repo", "namespace/:repo", "fullpath", "id", "user", "username", "group", "namespace", "repo", "branch", } var placeholderRE = regexp.MustCompile(`:(` + strings.Join(Placeholders, "|") + `)\b`) var placeholderNameRE = regexp.MustCompile(`^[A-Za-z](?:[A-Za-z0-9_-]*[A-Za-z0-9_])?$`) // jsonFieldHint is shared by both --field JSON failures so the guidance cannot // drift between them. const jsonFieldHint = `Use proper JSON syntax (e.g. -F 'key=["value1","value2"]') or -f to pass a literal string.` // legacyRawArrayRE matches the bracketed shorthand that --raw-field values were // once converted into arrays by, before array handling moved to --field only. // It exists to warn, not to parse: the value itself is sent literally. var legacyRawArrayRE = regexp.MustCompile(`^\[\s*([[:lower:]_]+(\s*,\s*[[:lower:]_]+)*)?\s*\]$`) // fillPlaceholders populates `:namespace` and `:repo` placeholders with values from the current repository. // When escapePath is true, substituted values are URL-encoded so they're safe as a single path segment; // callers expanding placeholders into request bodies or query values should pass false to preserve raw values. func fillPlaceholders(value string, opts *options, escapePath bool) (string, error) { re := placeholderRE if opts.placeholderRE != nil { re = opts.placeholderRE } if !re.MatchString(value) { return value, nil } maybeEscape := func(s string) string { if escapePath { return url.PathEscape(s) } return s } var err error filled := re.ReplaceAllStringFunc(value, func(m string) string { if custom, ok := opts.placeholders[m[1:]]; ok { return maybeEscape(custom) } switch m { case ":id": baseRepo, baseRepoErr := opts.baseRepo() if baseRepoErr != nil { err = baseRepoErr return "" } h, apiClientErr := opts.apiClient(baseRepo.RepoHost()) if apiClientErr != nil { err = apiClientErr return "" } project, e := baseRepo.Project(h.Lab()) if e == nil && project != nil { return strconv.FormatInt(project.ID, 10) } err = e return "" case ":group/:namespace/:repo", ":fullpath": baseRepo, baseRepoErr := opts.baseRepo() if baseRepoErr != nil { err = baseRepoErr return "" } return maybeEscape(baseRepo.FullName()) case ":namespace/:repo": baseRepo, baseRepoErr := opts.baseRepo() if baseRepoErr != nil { err = baseRepoErr return "" } return maybeEscape(baseRepo.RepoNamespace() + "/" + baseRepo.RepoName()) case ":group": baseRepo, baseRepoErr := opts.baseRepo() if baseRepoErr != nil { err = baseRepoErr return "" } return maybeEscape(baseRepo.RepoGroup()) case ":user", ":username": h, apiClientErr := opts.apiClient("") if apiClientErr != nil { err = apiClientErr return m } u, _, e := h.Lab().Users.CurrentUser() if e == nil && u != nil { return maybeEscape(u.Username) } err = e return m case ":namespace": baseRepo, baseRepoErr := opts.baseRepo() if baseRepoErr != nil { err = baseRepoErr return "" } return maybeEscape(baseRepo.RepoNamespace()) case ":repo": baseRepo, baseRepoErr := opts.baseRepo() if baseRepoErr != nil { err = baseRepoErr return "" } return maybeEscape(baseRepo.RepoName()) case ":branch": branch, e := opts.branch() if e != nil { err = e } return maybeEscape(branch) default: err = fmt.Errorf("invalid placeholder: %q", m) return "" } }) if err != nil { return value, err } return filled, nil } func printHeaders(w io.Writer, headers http.Header, colorize bool) { var names []string for name := range headers { if name == "Status" { continue } names = append(names, name) } sort.Strings(names) var headerColor, headerColorReset string if colorize { headerColor = "\x1b[1;34m" // bright blue headerColorReset = "\x1b[m" } for _, name := range names { fmt.Fprintf(w, "%s%s%s: %s\r\n", headerColor, name, headerColorReset, strings.Join(headers[name], ", ")) //nolint:forbidigo // w is a generic io.Writer, passed io.Discard in tests } } // methodForRequest is the method the request is sent with: absent --method, a // field, a form field, or --input makes it a POST rather than the flag's GET // default. resolveMethod is shared with the telemetry hook, so the two cannot // disagree about the rule; each site computes hasBody for itself. func (o *options) methodForRequest() string { return resolveMethod(o.requestMethod, o.requestMethodPassed, len(o.fields) > 0 || o.requestInputFile != "" || len(o.formFields) > 0) } // fieldsBecomeJSONBody reports whether the fields become a body, not a query. func (o *options) fieldsBecomeJSONBody(method string) bool { if o.requestInputFile != "" { // --input supplies the body, so the fields are query parameters. return false } return !isQueryMethod(method) } // parseFields returns the request parameters, plus the keys that came from // --raw-field in the order they were given. The caller needs that provenance to // warn about legacy bracketed values without parsing the flags a second time, // and a map alone cannot supply it: --field overwrites a --raw-field of the same // name, and map iteration has no order. With jsonBody, a bracketed name errors. func parseFields(opts *options, jsonBody bool) (map[string]any, []string, error) { params := make(map[string]any) rawKeys := make([]string, 0, len(opts.fields)) // A --raw-field never overrides a --field; the old two-pass read implied that. magicOwned := make(map[string]struct{}, len(opts.fields)) for _, f := range opts.fields { key, strValue, err := parseField(f.spec) if err != nil { return params, rawKeys, err } if jsonBody && strings.ContainsAny(key, "[]") { return params, rawKeys, bracketedFieldNameError(key) } value := any(strValue) if !f.raw { if value, err = magicFieldValue(strValue, opts); err != nil { return params, rawKeys, fmt.Errorf("error parsing %q value: %w", key, err) } } // A name ending in "[]" collects rather than holds one value, so there is // no override to apply: every value is sent. accumulates := strings.HasSuffix(key, "[]") if f.raw { rawKeys = append(rawKeys, key) if _, owned := magicOwned[key]; owned { continue } } else if !accumulates { magicOwned[key] = struct{}{} } addFieldParam(params, key, value) } if !jsonBody { if err := checkQueryKeyCollision(params); err != nil { return params, rawKeys, err } } return params, rawKeys, nil } // bracketedNameRE splits a name into its root and its bracketed segments. Root // and segments are both bracket-free, so a name the bracket syntax cannot // describe, such as "a[b", "a]" or "a[b]c", does not match. The guard in // parseFields is wider than this: it rejects any name holding a bracket at all. var bracketedNameRE = regexp.MustCompile(`^([^\[\]]+)((?:\[[^\[\]]*\])+)$`) // bracketedNameProblem is the clause every such error shares, whatever advice // follows it. Tests derive their expectations from it rather than respelling it. const bracketedNameProblem = "a field name containing a bracket is not supported in a JSON request body; pass the value as JSON" // bracketedFieldNameError reports a name a JSON body cannot carry. Where the // name has the shape its brackets describe, the advice is built from the user's // own segments, so it spells the request they were asking for rather than an // example they have to translate. A name that has no such shape gets the forms // that work and no invented key. func bracketedFieldNameError(key string) error { m := bracketedNameRE.FindStringSubmatch(key) if m == nil { return fmt.Errorf("field name %q: %s with -F, or use --input", key, bracketedNameProblem) } segments := strings.Split(strings.TrimSuffix(strings.TrimPrefix(m[2], "["), "]"), "][") return fmt.Errorf("field name %q: %s, for example -F '%s=%s', or use --input", key, bracketedNameProblem, m[1], jsonShapeOf(segments)) } // jsonShapeOf renders the value a name's bracketed segments describe, outermost // first: an empty segment is an array, a named one an object key. The innermost // value is a placeholder, since the name says nothing about the value's type. func jsonShapeOf(segments []string) string { if len(segments) == 0 { return `"..."` } head, rest := segments[0], segments[1:] if head == "" { if len(rest) == 0 { // An array of unknown element type, so no element is invented either. return "[...]" } return "[" + jsonShapeOf(rest) + "]" } // strconv.Quote, not the bare name: a segment may hold a quote of its own. return "{" + strconv.Quote(head) + ":" + jsonShapeOf(rest) + "}" } // addFieldParam stores one field's value. A name ending in "[]" accumulates in // flag order; any other, position[base_sha] included, stays last-wins. func addFieldParam(params map[string]any, key string, value any) { existing, seen := params[key] if !seen || !strings.HasSuffix(key, "[]") { params[key] = value return } if list, isList := existing.(queryList); isList { params[key] = append(list, value) return } params[key] = queryList{existing, value} } // queryWireKey is the query parameter a name addresses: a JSON array emits key[]=, // except where the name already carries those brackets, as parseQuery also reads // it. Computing "ids[][]" here would hide the clash between -F 'ids[]=[1,2]' and // -F 'ids=[3]', which both address ids[]. func queryWireKey(name string, value any) string { if _, isArray := value.([]any); isArray && !strings.HasSuffix(name, "[]") { return name + "[]" } return name } // checkQueryKeyCollision rejects two field names that emit one query parameter, // as -f 'ids[]=1' -F 'ids=[2,3]' does: map order would shuffle their values. func checkQueryKeyCollision(params map[string]any) error { owner := make(map[string]string, len(params)) for name, value := range params { wire := queryWireKey(name, value) prior, clash := owner[wire] if !clash { owner[wire] = name continue } // Order the two names so the message does not depend on map order. first, second := prior, name if second < first { first, second = second, first } return fmt.Errorf("field names %q and %q both address query parameter %q; spell the array one way, or use --input", first, second, wire) } return nil } // warnOnLegacyRawArrays prints a one-time hint per matching field when a // --raw-field value has the shape that used to be converted into a JSON array // on request bodies. The value is sent literally now, which the API usually // rejects outright, but an endpoint that accepts either a string or an array // would accept it silently as a single element. The hint makes that // discoverable. It goes to stderr, so scripted stdout consumers are unaffected. // // It reads the values parseFields already produced, walking rawKeys so the // warnings come out in flag order. Reading the parsed value is what makes this // honest: when a --field of the same name overrides a --raw-field, the literal // string is not what gets sent, so there is nothing to warn about. func (o *options) warnOnLegacyRawArrays(method string, params map[string]any, rawKeys []string) { if isQueryMethod(method) { // Query parameters were never array-converted, so nothing changed here. return } warned := make(map[string]bool, len(rawKeys)) for _, key := range rawKeys { if warned[key] { continue } value, isString := params[key].(string) if !isString || !legacyRawArrayRE.MatchString(value) { continue } warned[key] = true o.io.LogErrorf( "Warning: --raw-field %q is sent as the literal string %s. Use -F (--field) with JSON for an array, e.g. -F '%s=[\"value1\",\"value2\"]'.\n", key, value, key, ) } } func parseField(f string) (string, string, error) { idx := strings.IndexRune(f, '=') if idx == -1 { return f, "", fmt.Errorf("field %q requires a value separated by an '=' sign", f) } return f[0:idx], f[idx+1:], nil } func magicFieldValue(v string, opts *options) (any, error) { if strings.HasPrefix(v, "@") { return readUserFile(v[1:], opts.io.In) } if n, err := strconv.Atoi(v); err == nil { return n, nil } if strings.HasPrefix(v, "[") || strings.HasPrefix(v, "{") { // Decode with UseNumber so large integers keep full precision rather // than degrading through float64. dec := json.NewDecoder(strings.NewReader(v)) dec.UseNumber() var parsed any if err := dec.Decode(&parsed); err != nil { // The decoder's own message names the offending character, and the // caller already prefixes the field name, so an "invalid JSON" of // our own would be the third restatement of the same fact. return nil, fmt.Errorf("%w. %s", err, jsonFieldHint) } // Reject trailing data so a value like `[1,2]oops` still errors rather // than silently parsing only the leading JSON value. The decoder error // locates the offending character, so keep it rather than flattening // this to a bare "trailing data". if _, err := dec.Token(); !errors.Is(err, io.EOF) { if err == nil { err = errors.New("more than one JSON value") } return nil, fmt.Errorf("unexpected trailing data: %w. %s", err, jsonFieldHint) } return expandPlaceholdersIn(parsed, opts) } switch v { case "true": return true, nil case "false": return false, nil case "null": return nil, nil default: return fillPlaceholders(v, opts, false) } } // expandPlaceholdersIn expands placeholders in every string leaf and object key // of a JSON-decoded value, so `-F 'input={"projectPath":":fullpath"}'` behaves // the way the flag's documentation promises. // // Expansion runs after decoding, never before. Splicing an expanded value into // the raw JSON text would let a branch name containing a double quote, which // git accepts as a legal refname, inject JSON structure into an authenticated // request. Decoding first means the expanded text is an ordinary Go string that // json.Marshal re-escapes on the way out, and url.Values.Encode percent-encodes // on the query path. func expandPlaceholdersIn(v any, opts *options) (any, error) { switch t := v.(type) { case string: return fillPlaceholders(t, opts, false) case []any: for i, item := range t { expanded, err := expandPlaceholdersIn(item, opts) if err != nil { return nil, err } t[i] = expanded } return t, nil case map[string]any: // Build a new map rather than mutating during iteration: an expanded key // may collide with or replace an existing one. Such a collision is // rejected rather than resolved, because Go randomizes map iteration // order, so silently keeping one of the two would drop a field from the // request body non-deterministically between runs of the same command. out := make(map[string]any, len(t)) // Records which source key produced each expanded key, so the error can // name both culprits. Reporting only the key we happened to reach second // would make the message itself depend on iteration order, and for a // literal key colliding with a placeholder it would read "test expands to // test" whenever the literal came second. source := make(map[string]string, len(t)) for key, item := range t { expandedKey, err := fillPlaceholders(key, opts, false) if err != nil { return nil, err } if prior, taken := source[expandedKey]; taken { first, second := prior, key if first > second { first, second = second, first } return nil, fmt.Errorf( "keys %q and %q both expand to %q. Rename one of them so the request body is unambiguous", first, second, expandedKey, ) } source[expandedKey] = key expanded, err := expandPlaceholdersIn(item, opts) if err != nil { return nil, err } out[expandedKey] = expanded } return out, nil default: // Numbers, booleans and null carry no placeholders. return v, nil } } func readUserFile(fn string, stdin io.ReadCloser) ([]byte, error) { var r io.ReadCloser if fn == "-" { r = stdin } else { var err error r, err = os.Open(fn) if err != nil { return nil, err } } defer r.Close() return io.ReadAll(r) } func openUserFile(fn string, stdin io.ReadCloser) (io.ReadCloser, int64, error) { if fn == "-" { return stdin, -1, nil } r, err := os.Open(fn) if err != nil { return r, -1, err } s, err := os.Stat(fn) if err != nil { r.Close() return r, -1, err } return r, s.Size(), nil } func parseErrorResponse(r io.Reader, statusCode int) (io.Reader, string, error) { bodyCopy := &bytes.Buffer{} b, err := io.ReadAll(io.TeeReader(r, bodyCopy)) if err != nil { return r, "", err } var parsedBody struct { Message string Errors []json.RawMessage } err = json.Unmarshal(b, &parsedBody) if err != nil { // in cases where it's an object within an object we can try to parse it as is var t any err = json.Unmarshal(b, &t) if err != nil { return r, "", err } return bodyCopy, fmt.Sprintf("%+v", t), nil } if parsedBody.Message != "" { return bodyCopy, fmt.Sprintf("%s (HTTP %d)", parsedBody.Message, statusCode), nil } type errorMessage struct { Message string } var respErrors []string for _, rawErr := range parsedBody.Errors { if len(rawErr) == 0 { continue } if rawErr[0] == '{' { var objectError errorMessage err := json.Unmarshal(rawErr, &objectError) if err != nil { return r, "", err } respErrors = append(respErrors, objectError.Message) } else if rawErr[0] == '"' { var stringError string err := json.Unmarshal(rawErr, &stringError) if err != nil { return r, "", err } respErrors = append(respErrors, stringError) } } if len(respErrors) > 0 { return bodyCopy, strings.Join(respErrors, "\n"), nil } return bodyCopy, "", nil }