artifacts_uploader.go 3.57 KB
Newer Older
1
package helpers
2
3

import (
4
	"compress/gzip"
5
	"errors"
Kamil Trzciński's avatar
Kamil Trzciński committed
6
	"fmt"
7
8
	"io"
	"os"
9
	"path"
10
	"path/filepath"
11
12
	"time"

13
	"github.com/sirupsen/logrus"
14
	"github.com/urfave/cli"
15

16
17
18
19
	"gitlab.com/gitlab-org/gitlab-runner/common"
	"gitlab.com/gitlab-org/gitlab-runner/helpers/archives"
	"gitlab.com/gitlab-org/gitlab-runner/helpers/formatter"
	"gitlab.com/gitlab-org/gitlab-runner/network"
20
21
22
)

type ArtifactsUploaderCommand struct {
23
	common.JobCredentials
Kamil Trzcinski's avatar
Kamil Trzcinski committed
24
25
26
	fileArchiver
	retryHelper
	network common.Network
27

Kamil Trzciński's avatar
Kamil Trzciński committed
28
29
	Name     string                `long:"name" description:"The name of the archive"`
	ExpireIn string                `long:"expire-in" description:"When to expire artifacts"`
30
31
	Format   common.ArtifactFormat `long:"artifact-format" description:"Format of generated artifacts"`
	Type     string                `long:"artifact-type" description:"Type of generated artifacts"`
Kamil Trzciński's avatar
Kamil Trzciński committed
32
33
34
35
36
37
38
}

func (c *ArtifactsUploaderCommand) generateZipArchive(w *io.PipeWriter) {
	err := archives.CreateZipArchive(w, c.sortedFiles())
	w.CloseWithError(err)
}

39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
func (c *ArtifactsUploaderCommand) writeGzipFile(w *io.PipeWriter, fileInfo os.FileInfo) error {
	gz := gzip.NewWriter(w)
	gz.Header.Name = filepath.Base(fileInfo.Name())
	gz.Header.Comment = fileInfo.Name()
	gz.Header.ModTime = fileInfo.ModTime()
	defer gz.Close()

	file, err := os.Open(fileInfo.Name())
	if err != nil {
		return err
	}
	defer file.Close()

	_, err = io.Copy(gz, file)
	return err
}

func (c *ArtifactsUploaderCommand) generateGzipStream(w *io.PipeWriter) {
	var err error
	for _, fileInfo := range c.files {
		err = c.writeGzipFile(w, fileInfo)
		if err != nil {
			break
		}
	}
	w.CloseWithError(err)
}

Kamil Trzciński's avatar
Kamil Trzciński committed
67
68
69
70
71
func (c *ArtifactsUploaderCommand) createReadStream() (string, io.ReadCloser, error) {
	switch c.Format {
	case common.ArtifactFormatZip, "":
		pr, pw := io.Pipe()
		go c.generateZipArchive(pw)
Kamil Trzciński's avatar
Kamil Trzciński committed
72
		return path.Base(c.Name) + ".zip", pr, nil
Kamil Trzciński's avatar
Kamil Trzciński committed
73

74
	case common.ArtifactFormatGzip:
Kamil Trzciński's avatar
Kamil Trzciński committed
75
76
77
78
		if len(c.files) == 0 {
			return "", nil, errors.New("no file to upload")
		}

79
80
		pr, pw := io.Pipe()
		go c.generateGzipStream(pw)
Kamil Trzciński's avatar
Kamil Trzciński committed
81
		return c.Name + ".gz", pr, nil
Kamil Trzciński's avatar
Kamil Trzciński committed
82
83
84
85

	default:
		return "", nil, fmt.Errorf("unsupported archive format: %s", c.Format)
	}
86
87
}

Kamil Trzcinski's avatar
Kamil Trzcinski committed
88
func (c *ArtifactsUploaderCommand) createAndUpload() (bool, error) {
Kamil Trzciński's avatar
Kamil Trzciński committed
89
90
91
92
93
	artifactsName, stream, err := c.createReadStream()
	if err != nil {
		return false, err
	}
	defer stream.Close()
94
95

	// Create the archive
Kamil Trzciński's avatar
Kamil Trzciński committed
96
97
98
99
100
101
	options := common.ArtifactsOptions{
		BaseName: artifactsName,
		ExpireIn: c.ExpireIn,
		Format:   c.Format,
		Type:     c.Type,
	}
102

103
	// Upload the data
Kamil Trzciński's avatar
Kamil Trzciński committed
104
	switch c.network.UploadRawArtifacts(c.JobCredentials, stream, options) {
Kamil Trzcinski's avatar
Kamil Trzcinski committed
105
106
107
108
109
110
111
112
113
114
115
	case common.UploadSucceeded:
		return false, nil
	case common.UploadForbidden:
		return false, os.ErrPermission
	case common.UploadTooLarge:
		return false, errors.New("Too large")
	case common.UploadFailed:
		return true, os.ErrInvalid
	default:
		return false, os.ErrInvalid
	}
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
}

func (c *ArtifactsUploaderCommand) Execute(*cli.Context) {
	formatter.SetRunnerFormatter()

	if len(c.URL) == 0 || len(c.Token) == 0 {
		logrus.Fatalln("Missing runner credentials")
	}
	if c.ID <= 0 {
		logrus.Fatalln("Missing build ID")
	}

	// Enumerate files
	err := c.enumerate()
	if err != nil {
		logrus.Fatalln(err)
	}

	// If the upload fails, exit with a non-zero exit code to indicate an issue?
Kamil Trzcinski's avatar
Kamil Trzcinski committed
135
136
137
	err = c.doRetry(c.createAndUpload)
	if err != nil {
		logrus.Fatalln(err)
138
139
140
141
	}
}

func init() {
Kamil Trzcinski's avatar
Kamil Trzcinski committed
142
	common.RegisterCommand2("artifacts-uploader", "create and upload build artifacts (internal)", &ArtifactsUploaderCommand{
143
		network: network.NewGitLabClient(),
Kamil Trzcinski's avatar
Kamil Trzcinski committed
144
145
146
147
		retryHelper: retryHelper{
			Retry:     2,
			RetryTime: time.Second,
		},
148
		Name: "artifacts",
Kamil Trzcinski's avatar
Kamil Trzcinski committed
149
	})
150
}