Commit e859b829 authored by Kamil Trzciński's avatar Kamil Trzciński
Browse files

Add format/type test for artifacts_uploader

parent 359ba611
......@@ -3,7 +3,9 @@ package helpers
import (
"archive/zip"
"bytes"
"compress/gzip"
"io"
"io/ioutil"
"os"
"github.com/sirupsen/logrus"
......@@ -11,7 +13,10 @@ import (
"gitlab.com/gitlab-org/gitlab-runner/common"
)
const artifactsTestArchivedFile = "archive_file"
const (
artifactsTestArchivedFile = "archive_file"
artifactsTestArchivedFile2 = "archive_file2"
)
type testNetwork struct {
common.MockNetwork
......@@ -19,6 +24,10 @@ type testNetwork struct {
downloadCalled int
uploadState common.UploadState
uploadCalled int
uploadFormat common.ArtifactFormat
uploadName string
uploadType string
uploadedFiles []string
}
func (m *testNetwork) DownloadArtifacts(config common.JobCredentials, artifactsFile string) common.DownloadState {
......@@ -39,22 +48,76 @@ func (m *testNetwork) DownloadArtifacts(config common.JobCredentials, artifactsF
return m.downloadState
}
func (m *testNetwork) UploadRawArtifacts(config common.JobCredentials, reader io.Reader, options common.ArtifactsOptions) common.UploadState {
m.uploadCalled++
func (m *testNetwork) consumeZipUpload(config common.JobCredentials, reader io.Reader, options common.ArtifactsOptions) common.UploadState {
var buffer bytes.Buffer
io.Copy(&buffer, reader)
archive, err := zip.NewReader(bytes.NewReader(buffer.Bytes()), int64(buffer.Len()))
if err != nil {
logrus.Warningln(err)
return common.UploadForbidden
}
if m.uploadState == common.UploadSucceeded {
var buffer bytes.Buffer
io.Copy(&buffer, reader)
archive, err := zip.NewReader(bytes.NewReader(buffer.Bytes()), int64(buffer.Len()))
for _, file := range archive.File {
m.uploadedFiles = append(m.uploadedFiles, file.Name)
}
m.uploadFormat = common.ArtifactFormatZip
return m.uploadState
}
func (m *testNetwork) consumeGzipUpload(config common.JobCredentials, reader io.Reader, options common.ArtifactsOptions) common.UploadState {
var buffer bytes.Buffer
io.Copy(&buffer, reader)
stream := bytes.NewReader(buffer.Bytes())
gz, err := gzip.NewReader(stream)
gz.Multistream(false)
if err != nil {
logrus.Warningln("Invalid gzip stream")
return common.UploadForbidden
}
// Read multiple streams
for {
_, err = io.Copy(ioutil.Discard, gz)
if err != nil {
logrus.Warningln(err)
logrus.Warningln("Invalid gzip stream")
return common.UploadForbidden
}
if len(archive.File) != 1 || archive.File[0].Name != artifactsTestArchivedFile {
logrus.Warningln("Invalid archive:", len(archive.File))
m.uploadedFiles = append(m.uploadedFiles, gz.Name)
if gz.Reset(stream) == io.EOF {
break
}
gz.Multistream(false)
}
m.uploadFormat = common.ArtifactFormatGzip
return m.uploadState
}
func (m *testNetwork) UploadRawArtifacts(config common.JobCredentials, reader io.Reader, options common.ArtifactsOptions) common.UploadState {
m.uploadCalled++
if m.uploadState == common.UploadSucceeded {
m.uploadType = options.Type
m.uploadName = options.BaseName
switch options.Format {
case common.ArtifactFormatZip, common.ArtifactFormatDefault:
return m.consumeZipUpload(config, reader, options)
case common.ArtifactFormatGzip:
return m.consumeGzipUpload(config, reader, options)
default:
return common.UploadForbidden
}
}
return m.uploadState
}
......@@ -6,7 +6,6 @@ import (
"fmt"
"io"
"os"
"path"
"path/filepath"
"time"
......@@ -19,6 +18,8 @@ import (
"gitlab.com/gitlab-org/gitlab-runner/network"
)
const DefaultUploadName = "default"
type ArtifactsUploaderCommand struct {
common.JobCredentials
fileArchiver
......@@ -69,16 +70,21 @@ func (c *ArtifactsUploaderCommand) createReadStream() (string, io.ReadCloser, er
return "", nil, nil
}
name := filepath.Base(c.Name)
if name == "" || name == "." {
name = DefaultUploadName
}
switch c.Format {
case common.ArtifactFormatZip, "":
case common.ArtifactFormatZip, common.ArtifactFormatDefault:
pr, pw := io.Pipe()
go c.generateZipArchive(pw)
return path.Base(c.Name) + ".zip", pr, nil
return name + ".zip", pr, nil
case common.ArtifactFormatGzip:
pr, pw := io.Pipe()
go c.generateGzipStream(pw)
return c.Name + ".gz", pr, nil
return name + ".gz", pr, nil
default:
return "", nil, fmt.Errorf("unsupported archive format: %s", c.Format)
......
......@@ -35,8 +35,14 @@ func TestArtifactsUploaderTooLarge(t *testing.T) {
cmd := ArtifactsUploaderCommand{
JobCredentials: UploaderCredentials,
network: network,
fileArchiver: fileArchiver{
Paths: []string{artifactsTestArchivedFile},
},
}
ioutil.WriteFile(artifactsTestArchivedFile, nil, 0600)
defer os.Remove(artifactsTestArchivedFile)
removeHook := helpers.MakeFatalToPanic()
defer removeHook()
......@@ -54,8 +60,14 @@ func TestArtifactsUploaderForbidden(t *testing.T) {
cmd := ArtifactsUploaderCommand{
JobCredentials: UploaderCredentials,
network: network,
fileArchiver: fileArchiver{
Paths: []string{artifactsTestArchivedFile},
},
}
ioutil.WriteFile(artifactsTestArchivedFile, nil, 0600)
defer os.Remove(artifactsTestArchivedFile)
removeHook := helpers.MakeFatalToPanic()
defer removeHook()
......@@ -76,8 +88,14 @@ func TestArtifactsUploaderRetry(t *testing.T) {
retryHelper: retryHelper{
Retry: 2,
},
fileArchiver: fileArchiver{
Paths: []string{artifactsTestArchivedFile},
},
}
ioutil.WriteFile(artifactsTestArchivedFile, nil, 0600)
defer os.Remove(artifactsTestArchivedFile)
removeHook := helpers.MakeFatalToPanic()
defer removeHook()
......@@ -88,12 +106,37 @@ func TestArtifactsUploaderRetry(t *testing.T) {
assert.Equal(t, 3, network.uploadCalled)
}
func TestArtifactsUploaderSucceeded(t *testing.T) {
func TestArtifactsUploaderDefaultSucceeded(t *testing.T) {
network := &testNetwork{
uploadState: common.UploadSucceeded,
}
cmd := ArtifactsUploaderCommand{
JobCredentials: UploaderCredentials,
network: network,
fileArchiver: fileArchiver{
Paths: []string{artifactsTestArchivedFile},
},
}
ioutil.WriteFile(artifactsTestArchivedFile, nil, 0600)
defer os.Remove(artifactsTestArchivedFile)
cmd.Execute(nil)
assert.Equal(t, 1, network.uploadCalled)
assert.Equal(t, common.ArtifactFormatZip, network.uploadFormat)
assert.Equal(t, DefaultUploadName+".zip", network.uploadName)
assert.Empty(t, network.uploadType)
}
func TestArtifactsUploaderZipSucceeded(t *testing.T) {
network := &testNetwork{
uploadState: common.UploadSucceeded,
}
cmd := ArtifactsUploaderCommand{
JobCredentials: UploaderCredentials,
Format: common.ArtifactFormatZip,
Name: "my-release",
Type: "my-type",
network: network,
fileArchiver: fileArchiver{
Paths: []string{artifactsTestArchivedFile},
......@@ -105,6 +148,56 @@ func TestArtifactsUploaderSucceeded(t *testing.T) {
cmd.Execute(nil)
assert.Equal(t, 1, network.uploadCalled)
fi, _ := os.Stat(artifactsTestArchivedFile)
assert.NotNil(t, fi)
assert.Equal(t, common.ArtifactFormatZip, network.uploadFormat)
assert.Equal(t, "my-release.zip", network.uploadName)
assert.Equal(t, "my-type", network.uploadType)
assert.Contains(t, network.uploadedFiles, artifactsTestArchivedFile)
}
func TestArtifactsUploaderGzipSendsMultipleFiles(t *testing.T) {
network := &testNetwork{
uploadState: common.UploadSucceeded,
}
cmd := ArtifactsUploaderCommand{
JobCredentials: UploaderCredentials,
Format: common.ArtifactFormatGzip,
Name: "junit.xml",
Type: "junit",
network: network,
fileArchiver: fileArchiver{
Paths: []string{artifactsTestArchivedFile, artifactsTestArchivedFile2},
},
}
ioutil.WriteFile(artifactsTestArchivedFile, nil, 0600)
defer os.Remove(artifactsTestArchivedFile)
ioutil.WriteFile(artifactsTestArchivedFile2, nil, 0600)
defer os.Remove(artifactsTestArchivedFile)
cmd.Execute(nil)
assert.Equal(t, 1, network.uploadCalled)
assert.Equal(t, "junit.xml.gz", network.uploadName)
assert.Equal(t, common.ArtifactFormatGzip, network.uploadFormat)
assert.Equal(t, "junit", network.uploadType)
assert.Contains(t, network.uploadedFiles, artifactsTestArchivedFile)
assert.Contains(t, network.uploadedFiles, artifactsTestArchivedFile2)
}
func TestArtifactsUploaderNoFilesDoNotGenerateError(t *testing.T) {
network := &testNetwork{
uploadState: common.UploadSucceeded,
}
cmd := ArtifactsUploaderCommand{
JobCredentials: UploaderCredentials,
network: network,
fileArchiver: fileArchiver{},
}
removeHook := helpers.MakeFatalToPanic()
defer removeHook()
assert.NotPanics(t, func() {
cmd.Execute(nil)
})
}
......@@ -18,16 +18,20 @@ const fileArchiverAbsoluteFile = "/absolute.txt"
const fileArchiverRelativeFile = "../../../relative.txt"
func TestCacheArchiverAddingUntrackedFiles(t *testing.T) {
ioutil.WriteFile(fileArchiverUntrackedFile, nil, 0600)
defer os.Remove(fileArchiverUntrackedFile)
ioutil.WriteFile(artifactsTestArchivedFile, nil, 0600)
defer os.Remove(artifactsTestArchivedFile)
ioutil.WriteFile(artifactsTestArchivedFile2, nil, 0600)
defer os.Remove(artifactsTestArchivedFile2)
f := fileArchiver{
Untracked: true,
}
err := f.enumerate()
assert.NoError(t, err)
assert.Len(t, f.sortedFiles(), 1)
assert.Contains(t, f.sortedFiles(), fileArchiverUntrackedFile)
assert.Len(t, f.sortedFiles(), 2)
assert.Contains(t, f.sortedFiles(), artifactsTestArchivedFile)
assert.Contains(t, f.sortedFiles(), artifactsTestArchivedFile2)
}
func TestCacheArchiverAddingFile(t *testing.T) {
......
......@@ -194,8 +194,9 @@ func (when ArtifactWhen) OnFailure() bool {
type ArtifactFormat string
const (
ArtifactFormatZip ArtifactFormat = "zip"
ArtifactFormatGzip ArtifactFormat = "gzip"
ArtifactFormatDefault ArtifactFormat = ""
ArtifactFormatZip ArtifactFormat = "zip"
ArtifactFormatGzip ArtifactFormat = "gzip"
)
type Artifact struct {
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment