Commit ac85d037 authored by Kamil Trzciński's avatar Kamil Trzciński
Browse files

Add `artifact` format

parent d7be535a
......@@ -39,7 +39,7 @@ func (m *testNetwork) DownloadArtifacts(config common.JobCredentials, artifactsF
return m.downloadState
}
func (m *testNetwork) UploadRawArtifacts(config common.JobCredentials, reader io.Reader, baseName string, expireIn string) common.UploadState {
func (m *testNetwork) UploadRawArtifacts(config common.JobCredentials, reader io.Reader, options common.ArtifactsOptions) common.UploadState {
m.uploadCalled++
if m.uploadState == common.UploadSucceeded {
......
......@@ -2,6 +2,7 @@ package helpers
import (
"errors"
"fmt"
"io"
"os"
"path"
......@@ -22,24 +23,59 @@ type ArtifactsUploaderCommand struct {
retryHelper
network common.Network
Name string `long:"name" description:"The name of the archive"`
ExpireIn string `long:"expire-in" description:"When to expire artifacts"`
Name string `long:"name" description:"The name of the archive"`
ExpireIn string `long:"expire-in" description:"When to expire artifacts"`
Format common.ArtifactFormat `long:"format" description:"Format of generated artifacts"`
Type string `long:"type" description:"Type of generated artifacts"`
}
func (c *ArtifactsUploaderCommand) generateZipArchive(w *io.PipeWriter) {
err := archives.CreateZipArchive(w, c.sortedFiles())
w.CloseWithError(err)
}
func (c *ArtifactsUploaderCommand) createReadStream() (string, io.ReadCloser, error) {
artifactsName := path.Base(c.Name)
switch c.Format {
case common.ArtifactFormatZip, "":
pr, pw := io.Pipe()
go c.generateZipArchive(pw)
return artifactsName + ".zip", pr, nil
case common.ArtifactFormatRaw:
if len(c.files) == 0 {
return "", nil, errors.New("no file to upload")
}
if len(c.files) > 1 {
return "", nil, errors.New("raw format accepts only single file to upload")
}
file, err := os.Open(c.sortedFiles()[0])
return artifactsName, file, err
default:
return "", nil, fmt.Errorf("unsupported archive format: %s", c.Format)
}
}
func (c *ArtifactsUploaderCommand) createAndUpload() (bool, error) {
pr, pw := io.Pipe()
defer pr.Close()
artifactsName, stream, err := c.createReadStream()
if err != nil {
return false, err
}
defer stream.Close()
// Create the archive
go func() {
err := archives.CreateZipArchive(pw, c.sortedFiles())
pw.CloseWithError(err)
}()
artifactsName := path.Base(c.Name) + ".zip"
options := common.ArtifactsOptions{
BaseName: artifactsName,
ExpireIn: c.ExpireIn,
Format: c.Format,
Type: c.Type,
}
// Upload the data
switch c.network.UploadRawArtifacts(c.JobCredentials, pr, artifactsName, c.ExpireIn) {
switch c.network.UploadRawArtifacts(c.JobCredentials, stream, options) {
case common.UploadSucceeded:
return false, nil
case common.UploadForbidden:
......
......@@ -137,13 +137,13 @@ func (_m *MockNetwork) UploadArtifacts(config JobCredentials, artifactsFile stri
return r0
}
// UploadRawArtifacts provides a mock function with given fields: config, reader, baseName, expireIn
func (_m *MockNetwork) UploadRawArtifacts(config JobCredentials, reader io.Reader, baseName string, expireIn string) UploadState {
ret := _m.Called(config, reader, baseName, expireIn)
// UploadRawArtifacts provides a mock function with given fields: config, reader, options
func (_m *MockNetwork) UploadRawArtifacts(config JobCredentials, reader io.Reader, options ArtifactsOptions) UploadState {
ret := _m.Called(config, reader, options)
var r0 UploadState
if rf, ok := ret.Get(0).(func(JobCredentials, io.Reader, string, string) UploadState); ok {
r0 = rf(config, reader, baseName, expireIn)
if rf, ok := ret.Get(0).(func(JobCredentials, io.Reader, ArtifactsOptions) UploadState); ok {
r0 = rf(config, reader, options)
} else {
r0 = ret.Get(0).(UploadState)
}
......
......@@ -50,12 +50,13 @@ const (
)
type FeaturesInfo struct {
Variables bool `json:"variables"`
Image bool `json:"image"`
Services bool `json:"services"`
Artifacts bool `json:"features"`
Cache bool `json:"cache"`
Shared bool `json:"shared"`
Variables bool `json:"variables"`
Image bool `json:"image"`
Services bool `json:"services"`
Artifacts bool `json:"artifacts"`
Cache bool `json:"cache"`
Shared bool `json:"shared"`
ArtifactsFormat bool `json:"artifacts_format"`
}
type RegisterRunnerParameters struct {
......@@ -190,12 +191,21 @@ func (when ArtifactWhen) OnFailure() bool {
return when == ArtifactWhenOnFailure || when == ArtifactWhenAlways
}
type ArtifactFormat string
const (
ArtifactFormatZip ArtifactFormat = "zip"
ArtifactFormatRaw ArtifactFormat = "raw"
)
type Artifact struct {
Name string `json:"name"`
Untracked bool `json:"untracked"`
Paths ArtifactPaths `json:"paths"`
When ArtifactWhen `json:"when"`
ExpireIn string `json:"expire_in"`
Name string `json:"name"`
Untracked bool `json:"untracked"`
Paths ArtifactPaths `json:"paths"`
When ArtifactWhen `json:"when"`
Type string `json:"type"`
Format ArtifactFormat `json:"format"`
ExpireIn string `json:"expire_in"`
}
func (a Artifact) ShouldUpload(state error) bool {
......@@ -319,6 +329,13 @@ type UpdateJobInfo struct {
FailureReason JobFailureReason
}
type ArtifactsOptions struct {
BaseName string
ExpireIn string
Format ArtifactFormat
Type string
}
type FailuresCollector interface {
RecordFailure(reason JobFailureReason, runnerDescription string)
}
......@@ -348,7 +365,7 @@ type Network interface {
UpdateJob(config RunnerConfig, jobCredentials *JobCredentials, jobInfo UpdateJobInfo) UpdateState
PatchTrace(config RunnerConfig, jobCredentials *JobCredentials, tracePart JobTracePatch) UpdateState
DownloadArtifacts(config JobCredentials, artifactsFile string) DownloadState
UploadRawArtifacts(config JobCredentials, reader io.Reader, baseName string, expireIn string) UploadState
UploadRawArtifacts(config JobCredentials, reader io.Reader, options ArtifactsOptions) UploadState
UploadArtifacts(config JobCredentials, artifactsFile string) UploadState
ProcessJob(config RunnerConfig, buildCredentials *JobCredentials) JobTrace
}
......@@ -33,9 +33,9 @@ var apiRequestStatuses = prometheus.NewDesc(
type APIEndpoint string
const (
APIEndpointRequestJob APIEndpoint = "request_job"
APIEndpointUpdateJob APIEndpoint = "update_job"
APIEndpointPatchTrace APIEndpoint = "patch_trace"
APIEndpointRequestJob APIEndpoint = "request_job"
APIEndpointUpdateJob APIEndpoint = "update_job"
APIEndpointPatchTrace APIEndpoint = "patch_trace"
)
type apiRequestStatusPermutation struct {
......@@ -394,7 +394,7 @@ func (n *GitLabClient) createArtifactsForm(mpw *multipart.Writer, reader io.Read
return nil
}
func (n *GitLabClient) UploadRawArtifacts(config common.JobCredentials, reader io.Reader, baseName string, expireIn string) common.UploadState {
func (n *GitLabClient) UploadRawArtifacts(config common.JobCredentials, reader io.Reader, options common.ArtifactsOptions) common.UploadState {
pr, pw := io.Pipe()
defer pr.Close()
......@@ -403,15 +403,21 @@ func (n *GitLabClient) UploadRawArtifacts(config common.JobCredentials, reader i
go func() {
defer pw.Close()
defer mpw.Close()
err := n.createArtifactsForm(mpw, reader, baseName)
err := n.createArtifactsForm(mpw, reader, options.BaseName)
if err != nil {
pw.CloseWithError(err)
}
}()
query := url.Values{}
if expireIn != "" {
query.Set("expire_in", expireIn)
if options.ExpireIn != "" {
query.Set("expire_in", options.ExpireIn)
}
if options.Format != "" {
query.Set("format", string(options.Format))
}
if options.Type != "" {
query.Set("type", options.Type)
}
headers := make(http.Header)
......@@ -473,8 +479,10 @@ func (n *GitLabClient) UploadArtifacts(config common.JobCredentials, artifactsFi
return common.UploadFailed
}
baseName := filepath.Base(artifactsFile)
return n.UploadRawArtifacts(config, file, baseName, "")
options := common.ArtifactsOptions{
BaseName: filepath.Base(artifactsFile),
}
return n.UploadRawArtifacts(config, file, options)
}
func (n *GitLabClient) DownloadArtifacts(config common.JobCredentials, artifactsFile string) common.DownloadState {
......
......@@ -19,6 +19,7 @@ type AbstractShell struct {
func (b *AbstractShell) GetFeatures(features *common.FeaturesInfo) {
features.Artifacts = true
features.Cache = true
features.ArtifactsFormat = true
}
func (b *AbstractShell) writeCdBuildDir(w ShellWriter, info common.ShellScriptInfo) {
......@@ -498,6 +499,10 @@ func (b *AbstractShell) uploadArtifacts(w ShellWriter, info common.ShellScriptIn
info.Build.Token,
"--id",
strconv.Itoa(info.Build.ID),
"--format",
string(artifacts.Format),
"--type",
artifacts.Type,
}
// Create list of files to archive
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment