artifacts_uploader.go 3.53 KB
Newer Older
1
package helpers
2
3

import (
4
	"errors"
Kamil Trzciński's avatar
Kamil Trzciński committed
5
	"fmt"
6
7
	"io"
	"os"
8
	"path/filepath"
9
10
	"time"

11
	"github.com/sirupsen/logrus"
12
	"github.com/urfave/cli"
13

14
15
	"gitlab.com/gitlab-org/gitlab-runner/common"
	"gitlab.com/gitlab-org/gitlab-runner/helpers/archives"
16
	"gitlab.com/gitlab-org/gitlab-runner/log"
17
	"gitlab.com/gitlab-org/gitlab-runner/network"
18
19
)

20
21
const DefaultUploadName = "default"

22
type ArtifactsUploaderCommand struct {
23
	common.JobCredentials
Kamil Trzcinski's avatar
Kamil Trzcinski committed
24
25
26
	fileArchiver
	retryHelper
	network common.Network
27

Kamil Trzciński's avatar
Kamil Trzciński committed
28
29
	Name     string                `long:"name" description:"The name of the archive"`
	ExpireIn string                `long:"expire-in" description:"When to expire artifacts"`
30
31
	Format   common.ArtifactFormat `long:"artifact-format" description:"Format of generated artifacts"`
	Type     string                `long:"artifact-type" description:"Type of generated artifacts"`
Kamil Trzciński's avatar
Kamil Trzciński committed
32
33
34
35
36
37
38
}

func (c *ArtifactsUploaderCommand) generateZipArchive(w *io.PipeWriter) {
	err := archives.CreateZipArchive(w, c.sortedFiles())
	w.CloseWithError(err)
}

39
func (c *ArtifactsUploaderCommand) generateGzipStream(w *io.PipeWriter) {
40
	err := archives.CreateGzipArchive(w, c.sortedFiles())
41
42
43
	w.CloseWithError(err)
}

44
45
46
47
func (c *ArtifactsUploaderCommand) openRawStream() (io.ReadCloser, error) {
	fileNames := c.sortedFiles()
	if len(fileNames) > 1 {
		return nil, errors.New("only one file can be send as raw")
Kamil Trzciński's avatar
Kamil Trzciński committed
48
49
	}

50
	return os.Open(fileNames[0])
Kamil Trzciński's avatar
Kamil Trzciński committed
51
52
}

Kamil Trzciński's avatar
Kamil Trzciński committed
53
func (c *ArtifactsUploaderCommand) createReadStream() (string, io.ReadCloser, error) {
54
55
56
57
	if len(c.files) == 0 {
		return "", nil, nil
	}

58
59
60
61
62
	name := filepath.Base(c.Name)
	if name == "" || name == "." {
		name = DefaultUploadName
	}

Kamil Trzciński's avatar
Kamil Trzciński committed
63
	switch c.Format {
64
	case common.ArtifactFormatZip, common.ArtifactFormatDefault:
Kamil Trzciński's avatar
Kamil Trzciński committed
65
66
		pr, pw := io.Pipe()
		go c.generateZipArchive(pw)
67

68
		return name + ".zip", pr, nil
Kamil Trzciński's avatar
Kamil Trzciński committed
69

70
71
72
	case common.ArtifactFormatGzip:
		pr, pw := io.Pipe()
		go c.generateGzipStream(pw)
73

74
		return name + ".gz", pr, nil
Kamil Trzciński's avatar
Kamil Trzciński committed
75

Kamil Trzciński's avatar
Kamil Trzciński committed
76
	case common.ArtifactFormatRaw:
77
		file, err := c.openRawStream()
78

79
		return name, file, err
Kamil Trzciński's avatar
Kamil Trzciński committed
80

Kamil Trzciński's avatar
Kamil Trzciński committed
81
82
83
	default:
		return "", nil, fmt.Errorf("unsupported archive format: %s", c.Format)
	}
84
85
}

86
func (c *ArtifactsUploaderCommand) createAndUpload() error {
Kamil Trzciński's avatar
Kamil Trzciński committed
87
88
	artifactsName, stream, err := c.createReadStream()
	if err != nil {
89
		return err
Kamil Trzciński's avatar
Kamil Trzciński committed
90
	}
91
92
	if stream == nil {
		logrus.Errorln("No files to upload")
93
94

		return nil
95
	}
Kamil Trzciński's avatar
Kamil Trzciński committed
96
	defer stream.Close()
97
98

	// Create the archive
Kamil Trzciński's avatar
Kamil Trzciński committed
99
100
101
102
103
104
	options := common.ArtifactsOptions{
		BaseName: artifactsName,
		ExpireIn: c.ExpireIn,
		Format:   c.Format,
		Type:     c.Type,
	}
105

106
	// Upload the data
Kamil Trzciński's avatar
Kamil Trzciński committed
107
	switch c.network.UploadRawArtifacts(c.JobCredentials, stream, options) {
Kamil Trzcinski's avatar
Kamil Trzcinski committed
108
	case common.UploadSucceeded:
109
		return nil
Kamil Trzcinski's avatar
Kamil Trzcinski committed
110
	case common.UploadForbidden:
111
		return os.ErrPermission
Kamil Trzcinski's avatar
Kamil Trzcinski committed
112
	case common.UploadTooLarge:
113
		return errors.New("too large")
Kamil Trzcinski's avatar
Kamil Trzcinski committed
114
	case common.UploadFailed:
115
		return retryableErr{err: os.ErrInvalid}
Kamil Trzcinski's avatar
Kamil Trzcinski committed
116
	default:
117
		return os.ErrInvalid
Kamil Trzcinski's avatar
Kamil Trzcinski committed
118
	}
119
120
121
}

func (c *ArtifactsUploaderCommand) Execute(*cli.Context) {
122
	log.SetRunnerFormatter()
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137

	if len(c.URL) == 0 || len(c.Token) == 0 {
		logrus.Fatalln("Missing runner credentials")
	}
	if c.ID <= 0 {
		logrus.Fatalln("Missing build ID")
	}

	// Enumerate files
	err := c.enumerate()
	if err != nil {
		logrus.Fatalln(err)
	}

	// If the upload fails, exit with a non-zero exit code to indicate an issue?
Kamil Trzcinski's avatar
Kamil Trzcinski committed
138
139
140
	err = c.doRetry(c.createAndUpload)
	if err != nil {
		logrus.Fatalln(err)
141
142
143
144
	}
}

func init() {
Kamil Trzcinski's avatar
Kamil Trzcinski committed
145
	common.RegisterCommand2("artifacts-uploader", "create and upload build artifacts (internal)", &ArtifactsUploaderCommand{
146
		network: network.NewGitLabClient(),
Kamil Trzcinski's avatar
Kamil Trzcinski committed
147
148
149
150
		retryHelper: retryHelper{
			Retry:     2,
			RetryTime: time.Second,
		},
151
		Name: "artifacts",
Kamil Trzcinski's avatar
Kamil Trzcinski committed
152
	})
153
}