artifacts_uploader.go 4.16 KB
Newer Older
1
package helpers
2
3

import (
4
	"errors"
Kamil Trzciński's avatar
Kamil Trzciński committed
5
	"fmt"
6
7
	"io"
	"os"
8
	"path/filepath"
9

10
	"github.com/sirupsen/logrus"
11
	"github.com/urfave/cli"
12

13
14
	"gitlab.com/gitlab-org/gitlab-runner/common"
	"gitlab.com/gitlab-org/gitlab-runner/helpers/archives"
15
	"gitlab.com/gitlab-org/gitlab-runner/helpers/retry"
16
	"gitlab.com/gitlab-org/gitlab-runner/log"
17
	"gitlab.com/gitlab-org/gitlab-runner/network"
18
19
)

20
21
22
23
24
25
26
27
28
29
const (
	DefaultUploadName       = "default"
	defaultTries            = 3
	serviceUnavailableTries = 6
)

var (
	errServiceUnavailable = errors.New("service unavailable")
	errTooLarge           = errors.New("too large")
)
30

31
type ArtifactsUploaderCommand struct {
32
	common.JobCredentials
Kamil Trzcinski's avatar
Kamil Trzcinski committed
33
34
	fileArchiver
	network common.Network
35

Kamil Trzciński's avatar
Kamil Trzciński committed
36
37
	Name     string                `long:"name" description:"The name of the archive"`
	ExpireIn string                `long:"expire-in" description:"When to expire artifacts"`
38
39
	Format   common.ArtifactFormat `long:"artifact-format" description:"Format of generated artifacts"`
	Type     string                `long:"artifact-type" description:"Type of generated artifacts"`
Kamil Trzciński's avatar
Kamil Trzciński committed
40
41
42
43
44
45
46
}

func (c *ArtifactsUploaderCommand) generateZipArchive(w *io.PipeWriter) {
	err := archives.CreateZipArchive(w, c.sortedFiles())
	w.CloseWithError(err)
}

47
func (c *ArtifactsUploaderCommand) generateGzipStream(w *io.PipeWriter) {
48
	err := archives.CreateGzipArchive(w, c.sortedFiles())
49
50
51
	w.CloseWithError(err)
}

52
53
54
55
func (c *ArtifactsUploaderCommand) openRawStream() (io.ReadCloser, error) {
	fileNames := c.sortedFiles()
	if len(fileNames) > 1 {
		return nil, errors.New("only one file can be send as raw")
Kamil Trzciński's avatar
Kamil Trzciński committed
56
57
	}

58
	return os.Open(fileNames[0])
Kamil Trzciński's avatar
Kamil Trzciński committed
59
60
}

Kamil Trzciński's avatar
Kamil Trzciński committed
61
func (c *ArtifactsUploaderCommand) createReadStream() (string, io.ReadCloser, error) {
62
63
64
65
	if len(c.files) == 0 {
		return "", nil, nil
	}

66
67
68
69
70
	name := filepath.Base(c.Name)
	if name == "" || name == "." {
		name = DefaultUploadName
	}

Kamil Trzciński's avatar
Kamil Trzciński committed
71
	switch c.Format {
72
	case common.ArtifactFormatZip, common.ArtifactFormatDefault:
Kamil Trzciński's avatar
Kamil Trzciński committed
73
74
		pr, pw := io.Pipe()
		go c.generateZipArchive(pw)
75

76
		return name + ".zip", pr, nil
Kamil Trzciński's avatar
Kamil Trzciński committed
77

78
79
80
	case common.ArtifactFormatGzip:
		pr, pw := io.Pipe()
		go c.generateGzipStream(pw)
81

82
		return name + ".gz", pr, nil
Kamil Trzciński's avatar
Kamil Trzciński committed
83

Kamil Trzciński's avatar
Kamil Trzciński committed
84
	case common.ArtifactFormatRaw:
85
		file, err := c.openRawStream()
86

87
		return name, file, err
Kamil Trzciński's avatar
Kamil Trzciński committed
88

Kamil Trzciński's avatar
Kamil Trzciński committed
89
90
91
	default:
		return "", nil, fmt.Errorf("unsupported archive format: %s", c.Format)
	}
92
93
}

94
func (c *ArtifactsUploaderCommand) Run() error {
Kamil Trzciński's avatar
Kamil Trzciński committed
95
96
	artifactsName, stream, err := c.createReadStream()
	if err != nil {
97
		return err
Kamil Trzciński's avatar
Kamil Trzciński committed
98
	}
99
100
	if stream == nil {
		logrus.Errorln("No files to upload")
101
102

		return nil
103
	}
Kamil Trzciński's avatar
Kamil Trzciński committed
104
	defer stream.Close()
105
106

	// Create the archive
Kamil Trzciński's avatar
Kamil Trzciński committed
107
108
109
110
111
112
	options := common.ArtifactsOptions{
		BaseName: artifactsName,
		ExpireIn: c.ExpireIn,
		Format:   c.Format,
		Type:     c.Type,
	}
113

114
	// Upload the data
Kamil Trzciński's avatar
Kamil Trzciński committed
115
	switch c.network.UploadRawArtifacts(c.JobCredentials, stream, options) {
Kamil Trzcinski's avatar
Kamil Trzcinski committed
116
	case common.UploadSucceeded:
117
		return nil
Kamil Trzcinski's avatar
Kamil Trzcinski committed
118
	case common.UploadForbidden:
119
		return os.ErrPermission
Kamil Trzcinski's avatar
Kamil Trzcinski committed
120
	case common.UploadTooLarge:
121
		return errTooLarge
Kamil Trzcinski's avatar
Kamil Trzcinski committed
122
	case common.UploadFailed:
123
		return retryableErr{err: os.ErrInvalid}
124
125
	case common.UploadServiceUnavailable:
		return retryableErr{err: errServiceUnavailable}
Kamil Trzcinski's avatar
Kamil Trzcinski committed
126
	default:
127
		return os.ErrInvalid
Kamil Trzcinski's avatar
Kamil Trzcinski committed
128
	}
129
130
}

131
func (c *ArtifactsUploaderCommand) ShouldRetry(tries int, err error) bool {
132
133
	var errAs retryableErr
	if !errors.As(err, &errAs) {
134
135
136
137
		return false
	}

	maxTries := defaultTries
138
	if errors.Is(errAs, errServiceUnavailable) {
139
140
141
142
143
144
145
146
147
148
		maxTries = serviceUnavailableTries
	}

	if tries >= maxTries {
		return false
	}

	return true
}

149
func (c *ArtifactsUploaderCommand) Execute(*cli.Context) {
150
	log.SetRunnerFormatter()
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165

	if len(c.URL) == 0 || len(c.Token) == 0 {
		logrus.Fatalln("Missing runner credentials")
	}
	if c.ID <= 0 {
		logrus.Fatalln("Missing build ID")
	}

	// Enumerate files
	err := c.enumerate()
	if err != nil {
		logrus.Fatalln(err)
	}

	// If the upload fails, exit with a non-zero exit code to indicate an issue?
166
167
168
	logger := logrus.WithField("context", "artifacts-uploader")
	retryable := retry.New(retry.WithLogrus(c, logger))
	err = retryable.Run()
Kamil Trzcinski's avatar
Kamil Trzcinski committed
169
170
	if err != nil {
		logrus.Fatalln(err)
171
172
173
174
	}
}

func init() {
Pedro Pombeiro's avatar
Pedro Pombeiro committed
175
176
177
178
179
180
181
182
	common.RegisterCommand2(
		"artifacts-uploader",
		"create and upload build artifacts (internal)",
		&ArtifactsUploaderCommand{
			network: network.NewGitLabClient(),
			Name:    "artifacts",
		},
	)
183
}