Commit dc3ffeba authored by Kamil Trzciński's avatar Kamil Trzciński
Browse files

Merge branch 'ajwalker/cache-perf/interface-link' into 'master'

Teach artifact/cache commands about the archive interface

See merge request gitlab-org/gitlab-runner!2467
parents 575a5585 c3c62fd3
package helpers
import (
"context"
"io/ioutil"
"os"
"time"
......@@ -8,8 +9,8 @@ import (
"github.com/sirupsen/logrus"
"github.com/urfave/cli"
"gitlab.com/gitlab-org/gitlab-runner/commands/helpers/archive"
"gitlab.com/gitlab-org/gitlab-runner/common"
"gitlab.com/gitlab-org/gitlab-runner/helpers/archives"
"gitlab.com/gitlab-org/gitlab-runner/log"
"gitlab.com/gitlab-org/gitlab-runner/network"
)
......@@ -49,9 +50,14 @@ func (c *ArtifactsDownloaderCommand) download(file string, retry int) error {
}
}
func (c *ArtifactsDownloaderCommand) Execute(context *cli.Context) {
func (c *ArtifactsDownloaderCommand) Execute(cliContext *cli.Context) {
log.SetRunnerFormatter()
wd, err := os.Getwd()
if err != nil {
logrus.Fatalln("Unable to get working directory")
}
if c.URL == "" || c.Token == "" {
logrus.Fatalln("Missing runner credentials")
}
......@@ -75,13 +81,39 @@ func (c *ArtifactsDownloaderCommand) Execute(context *cli.Context) {
logrus.Fatalln(err)
}
f, size, err := openZip(file.Name())
if err != nil {
logrus.Fatalln(err)
}
defer f.Close()
extractor, err := archive.NewExtractor(archive.Zip, f, size, wd)
if err != nil {
logrus.Fatalln(err)
}
// Extract artifacts file
err = archives.ExtractZipFile(file.Name())
err = extractor.Extract(context.Background())
if err != nil {
logrus.Fatalln(err)
}
}
func openZip(filename string) (*os.File, int64, error) {
f, err := os.Open(filename)
if err != nil {
return nil, 0, err
}
fi, err := f.Stat()
if err != nil {
f.Close()
return nil, 0, err
}
return f, fi.Size(), nil
}
func init() {
common.RegisterCommand2(
"artifacts-downloader",
......
......@@ -112,7 +112,10 @@ func (m *testNetwork) consumeGzipUpload(reader io.Reader) common.UploadState {
}
func (m *testNetwork) consumeRawUpload(reader io.Reader) common.UploadState {
_, _ = io.Copy(ioutil.Discard, reader)
_, err := io.Copy(ioutil.Discard, reader)
if err != nil {
return common.UploadFailed
}
m.uploadedFiles = append(m.uploadedFiles, "raw")
m.uploadFormat = common.ArtifactFormatRaw
......
package helpers
import (
"context"
"errors"
"fmt"
"io"
"os"
"path/filepath"
......@@ -10,8 +10,8 @@ import (
"github.com/sirupsen/logrus"
"github.com/urfave/cli"
"gitlab.com/gitlab-org/gitlab-runner/commands/helpers/archive"
"gitlab.com/gitlab-org/gitlab-runner/common"
"gitlab.com/gitlab-org/gitlab-runner/helpers/archives"
"gitlab.com/gitlab-org/gitlab-runner/helpers/retry"
"gitlab.com/gitlab-org/gitlab-runner/log"
"gitlab.com/gitlab-org/gitlab-runner/network"
......@@ -39,23 +39,20 @@ type ArtifactsUploaderCommand struct {
Type string `long:"artifact-type" description:"Type of generated artifacts"`
}
func (c *ArtifactsUploaderCommand) generateZipArchive(w *io.PipeWriter) {
err := archives.CreateZipArchive(w, c.sortedFiles())
_ = w.CloseWithError(err)
}
func (c *ArtifactsUploaderCommand) artifactFilename(name string, format common.ArtifactFormat) string {
name = filepath.Base(name)
if name == "" || name == "." {
name = DefaultUploadName
}
func (c *ArtifactsUploaderCommand) generateGzipStream(w *io.PipeWriter) {
err := archives.CreateGzipArchive(w, c.sortedFiles())
_ = w.CloseWithError(err)
}
switch format {
case common.ArtifactFormatZip:
return name + ".zip"
func (c *ArtifactsUploaderCommand) openRawStream() (io.ReadCloser, error) {
fileNames := c.sortedFiles()
if len(fileNames) > 1 {
return nil, errors.New("only one file can be send as raw")
case common.ArtifactFormatGzip:
return name + ".gz"
}
return os.Open(fileNames[0])
return name
}
func (c *ArtifactsUploaderCommand) createReadStream() (string, io.ReadCloser, error) {
......@@ -63,32 +60,26 @@ func (c *ArtifactsUploaderCommand) createReadStream() (string, io.ReadCloser, er
return "", nil, nil
}
name := filepath.Base(c.Name)
if name == "" || name == "." {
name = DefaultUploadName
format := c.Format
if format == common.ArtifactFormatDefault {
format = common.ArtifactFormatZip
}
switch c.Format {
case common.ArtifactFormatZip, common.ArtifactFormatDefault:
filename := c.artifactFilename(c.Name, format)
pr, pw := io.Pipe()
go c.generateZipArchive(pw)
return name + ".zip", pr, nil
case common.ArtifactFormatGzip:
pr, pw := io.Pipe()
go c.generateGzipStream(pw)
return name + ".gz", pr, nil
case common.ArtifactFormatRaw:
file, err := c.openRawStream()
archiver, err := archive.NewArchiver(archive.Format(format), pw, c.wd, archive.DefaultCompression)
if err != nil {
_ = pr.CloseWithError(err)
return filename, nil, err
}
return name, file, err
go func() {
err := archiver.Archive(context.Background(), c.files)
_ = pw.CloseWithError(err)
}()
default:
return "", nil, fmt.Errorf("unsupported archive format: %s", c.Format)
}
return filename, pr, nil
}
func (c *ArtifactsUploaderCommand) Run() error {
......
package helpers
import (
"context"
"io/ioutil"
"net/http"
"os"
"path/filepath"
......@@ -10,8 +12,8 @@ import (
"github.com/sirupsen/logrus"
"github.com/urfave/cli"
"gitlab.com/gitlab-org/gitlab-runner/commands/helpers/archive"
"gitlab.com/gitlab-org/gitlab-runner/common"
"gitlab.com/gitlab-org/gitlab-runner/helpers/archives"
url_helpers "gitlab.com/gitlab-org/gitlab-runner/helpers/url"
"gitlab.com/gitlab-org/gitlab-runner/log"
)
......@@ -66,6 +68,40 @@ func (c *CacheArchiverCommand) upload(_ int) error {
return retryOnServerError(resp)
}
func (c *CacheArchiverCommand) createZipFile(filename string) error {
err := os.MkdirAll(filepath.Dir(filename), 0700)
if err != nil {
return err
}
f, err := ioutil.TempFile(filepath.Dir(filename), "archive_")
if err != nil {
return err
}
defer os.Remove(f.Name())
defer f.Close()
logrus.Debugln("Temporary file:", f.Name())
archiver, err := archive.NewArchiver(archive.Zip, f, c.wd, archive.DefaultCompression)
if err != nil {
return err
}
// Create archive
err = archiver.Archive(context.Background(), c.files)
if err != nil {
return err
}
err = f.Close()
if err != nil {
return err
}
return os.Rename(f.Name(), filename)
}
func (c *CacheArchiverCommand) Execute(*cli.Context) {
log.SetRunnerFormatter()
......@@ -87,7 +123,7 @@ func (c *CacheArchiverCommand) Execute(*cli.Context) {
}
// Create archive
err = archives.CreateZipFile(c.File, c.sortedFiles())
err = c.createZipFile(c.File)
if err != nil {
logrus.Fatalln(err)
}
......
package helpers
import (
"context"
"io"
"io/ioutil"
"net/http"
......@@ -11,8 +12,8 @@ import (
"github.com/sirupsen/logrus"
"github.com/urfave/cli"
"gitlab.com/gitlab-org/gitlab-runner/commands/helpers/archive"
"gitlab.com/gitlab-org/gitlab-runner/common"
"gitlab.com/gitlab-org/gitlab-runner/helpers/archives"
url_helpers "gitlab.com/gitlab-org/gitlab-runner/helpers/url"
"gitlab.com/gitlab-org/gitlab-runner/log"
)
......@@ -105,9 +106,14 @@ func (c *CacheExtractorCommand) getCache() (*http.Response, error) {
return resp, retryOnServerError(resp)
}
func (c *CacheExtractorCommand) Execute(context *cli.Context) {
func (c *CacheExtractorCommand) Execute(cliContext *cli.Context) {
log.SetRunnerFormatter()
wd, err := os.Getwd()
if err != nil {
logrus.Fatalln("Unable to get working directory")
}
if c.File == "" {
logrus.Fatalln("Missing cache file")
}
......@@ -123,8 +129,22 @@ func (c *CacheExtractorCommand) Execute(context *cli.Context) {
"Instead a local version of cache will be extracted.")
}
err := archives.ExtractZipFile(c.File)
if err != nil && !os.IsNotExist(err) {
f, size, err := openZip(c.File)
if os.IsNotExist(err) {
return
}
if err != nil {
logrus.Fatalln(err)
}
defer f.Close()
extractor, err := archive.NewExtractor(archive.Zip, f, size, wd)
if err != nil {
logrus.Fatalln(err)
}
err = extractor.Extract(context.Background())
if err != nil {
logrus.Fatalln(err)
}
}
......
......@@ -3,9 +3,7 @@ package archives
import (
"archive/zip"
"io"
"io/ioutil"
"os"
"path/filepath"
"github.com/sirupsen/logrus"
)
......@@ -103,34 +101,3 @@ func CreateZipArchive(w io.Writer, fileNames []string) error {
return nil
}
func CreateZipFile(fileName string, fileNames []string) error {
// create directories to store archive
err := os.MkdirAll(filepath.Dir(fileName), 0700)
if err != nil {
return err
}
tempFile, err := ioutil.TempFile(filepath.Dir(fileName), "archive_")
if err != nil {
return err
}
defer func() {
_ = tempFile.Close()
_ = os.Remove(tempFile.Name())
}()
logrus.Debugln("Temporary file:", tempFile.Name())
err = CreateZipArchive(tempFile, fileNames)
if err != nil {
return err
}
_ = tempFile.Close()
err = os.Rename(tempFile.Name(), fileName)
if err != nil {
return err
}
return nil
}
......@@ -116,8 +116,14 @@ func TestZipCreate(t *testing.T) {
createTestPipe(t, multiBytes),
"non_existing_file.txt",
}
err := CreateZipFile(fileName, paths)
f, err := os.Create(fileName)
require.NoError(t, err)
defer f.Close()
err = CreateZipArchive(f, paths)
require.NoError(t, err)
require.NoError(t, f.Close())
archive, err := zip.OpenReader(fileName)
require.NoError(t, err)
......@@ -158,8 +164,14 @@ func TestZipCreateWithGitPath(t *testing.T) {
createTestGitPathFile(t, singleByte),
createTestGitPathFile(t, multiBytes),
}
err := CreateZipFile(fileName, paths)
f, err := os.Create(fileName)
require.NoError(t, err)
defer f.Close()
err = CreateZipArchive(f, paths)
require.NoError(t, err)
require.NoError(t, f.Close())
assert.Contains(t, buf.String(), "Part of .git directory is on the list of files to archive")
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment