Commit dc3ffeba authored by Kamil Trzciński's avatar Kamil Trzciński
Browse files

Merge branch 'ajwalker/cache-perf/interface-link' into 'master'

Teach artifact/cache commands about the archive interface

See merge request gitlab-org/gitlab-runner!2467
parents 575a5585 c3c62fd3
package helpers package helpers
import ( import (
"context"
"io/ioutil" "io/ioutil"
"os" "os"
"time" "time"
...@@ -8,8 +9,8 @@ import ( ...@@ -8,8 +9,8 @@ import (
"github.com/sirupsen/logrus" "github.com/sirupsen/logrus"
"github.com/urfave/cli" "github.com/urfave/cli"
"gitlab.com/gitlab-org/gitlab-runner/commands/helpers/archive"
"gitlab.com/gitlab-org/gitlab-runner/common" "gitlab.com/gitlab-org/gitlab-runner/common"
"gitlab.com/gitlab-org/gitlab-runner/helpers/archives"
"gitlab.com/gitlab-org/gitlab-runner/log" "gitlab.com/gitlab-org/gitlab-runner/log"
"gitlab.com/gitlab-org/gitlab-runner/network" "gitlab.com/gitlab-org/gitlab-runner/network"
) )
...@@ -49,9 +50,14 @@ func (c *ArtifactsDownloaderCommand) download(file string, retry int) error { ...@@ -49,9 +50,14 @@ func (c *ArtifactsDownloaderCommand) download(file string, retry int) error {
} }
} }
func (c *ArtifactsDownloaderCommand) Execute(context *cli.Context) { func (c *ArtifactsDownloaderCommand) Execute(cliContext *cli.Context) {
log.SetRunnerFormatter() log.SetRunnerFormatter()
wd, err := os.Getwd()
if err != nil {
logrus.Fatalln("Unable to get working directory")
}
if c.URL == "" || c.Token == "" { if c.URL == "" || c.Token == "" {
logrus.Fatalln("Missing runner credentials") logrus.Fatalln("Missing runner credentials")
} }
...@@ -75,13 +81,39 @@ func (c *ArtifactsDownloaderCommand) Execute(context *cli.Context) { ...@@ -75,13 +81,39 @@ func (c *ArtifactsDownloaderCommand) Execute(context *cli.Context) {
logrus.Fatalln(err) logrus.Fatalln(err)
} }
f, size, err := openZip(file.Name())
if err != nil {
logrus.Fatalln(err)
}
defer f.Close()
extractor, err := archive.NewExtractor(archive.Zip, f, size, wd)
if err != nil {
logrus.Fatalln(err)
}
// Extract artifacts file // Extract artifacts file
err = archives.ExtractZipFile(file.Name()) err = extractor.Extract(context.Background())
if err != nil { if err != nil {
logrus.Fatalln(err) logrus.Fatalln(err)
} }
} }
func openZip(filename string) (*os.File, int64, error) {
f, err := os.Open(filename)
if err != nil {
return nil, 0, err
}
fi, err := f.Stat()
if err != nil {
f.Close()
return nil, 0, err
}
return f, fi.Size(), nil
}
func init() { func init() {
common.RegisterCommand2( common.RegisterCommand2(
"artifacts-downloader", "artifacts-downloader",
......
...@@ -112,7 +112,10 @@ func (m *testNetwork) consumeGzipUpload(reader io.Reader) common.UploadState { ...@@ -112,7 +112,10 @@ func (m *testNetwork) consumeGzipUpload(reader io.Reader) common.UploadState {
} }
func (m *testNetwork) consumeRawUpload(reader io.Reader) common.UploadState { func (m *testNetwork) consumeRawUpload(reader io.Reader) common.UploadState {
_, _ = io.Copy(ioutil.Discard, reader) _, err := io.Copy(ioutil.Discard, reader)
if err != nil {
return common.UploadFailed
}
m.uploadedFiles = append(m.uploadedFiles, "raw") m.uploadedFiles = append(m.uploadedFiles, "raw")
m.uploadFormat = common.ArtifactFormatRaw m.uploadFormat = common.ArtifactFormatRaw
......
package helpers package helpers
import ( import (
"context"
"errors" "errors"
"fmt"
"io" "io"
"os" "os"
"path/filepath" "path/filepath"
...@@ -10,8 +10,8 @@ import ( ...@@ -10,8 +10,8 @@ import (
"github.com/sirupsen/logrus" "github.com/sirupsen/logrus"
"github.com/urfave/cli" "github.com/urfave/cli"
"gitlab.com/gitlab-org/gitlab-runner/commands/helpers/archive"
"gitlab.com/gitlab-org/gitlab-runner/common" "gitlab.com/gitlab-org/gitlab-runner/common"
"gitlab.com/gitlab-org/gitlab-runner/helpers/archives"
"gitlab.com/gitlab-org/gitlab-runner/helpers/retry" "gitlab.com/gitlab-org/gitlab-runner/helpers/retry"
"gitlab.com/gitlab-org/gitlab-runner/log" "gitlab.com/gitlab-org/gitlab-runner/log"
"gitlab.com/gitlab-org/gitlab-runner/network" "gitlab.com/gitlab-org/gitlab-runner/network"
...@@ -39,23 +39,20 @@ type ArtifactsUploaderCommand struct { ...@@ -39,23 +39,20 @@ type ArtifactsUploaderCommand struct {
Type string `long:"artifact-type" description:"Type of generated artifacts"` Type string `long:"artifact-type" description:"Type of generated artifacts"`
} }
func (c *ArtifactsUploaderCommand) generateZipArchive(w *io.PipeWriter) { func (c *ArtifactsUploaderCommand) artifactFilename(name string, format common.ArtifactFormat) string {
err := archives.CreateZipArchive(w, c.sortedFiles()) name = filepath.Base(name)
_ = w.CloseWithError(err) if name == "" || name == "." {
} name = DefaultUploadName
}
func (c *ArtifactsUploaderCommand) generateGzipStream(w *io.PipeWriter) { switch format {
err := archives.CreateGzipArchive(w, c.sortedFiles()) case common.ArtifactFormatZip:
_ = w.CloseWithError(err) return name + ".zip"
}
func (c *ArtifactsUploaderCommand) openRawStream() (io.ReadCloser, error) { case common.ArtifactFormatGzip:
fileNames := c.sortedFiles() return name + ".gz"
if len(fileNames) > 1 {
return nil, errors.New("only one file can be send as raw")
} }
return name
return os.Open(fileNames[0])
} }
func (c *ArtifactsUploaderCommand) createReadStream() (string, io.ReadCloser, error) { func (c *ArtifactsUploaderCommand) createReadStream() (string, io.ReadCloser, error) {
...@@ -63,32 +60,26 @@ func (c *ArtifactsUploaderCommand) createReadStream() (string, io.ReadCloser, er ...@@ -63,32 +60,26 @@ func (c *ArtifactsUploaderCommand) createReadStream() (string, io.ReadCloser, er
return "", nil, nil return "", nil, nil
} }
name := filepath.Base(c.Name) format := c.Format
if name == "" || name == "." { if format == common.ArtifactFormatDefault {
name = DefaultUploadName format = common.ArtifactFormatZip
} }
switch c.Format { filename := c.artifactFilename(c.Name, format)
case common.ArtifactFormatZip, common.ArtifactFormatDefault: pr, pw := io.Pipe()
pr, pw := io.Pipe()
go c.generateZipArchive(pw)
return name + ".zip", pr, nil archiver, err := archive.NewArchiver(archive.Format(format), pw, c.wd, archive.DefaultCompression)
if err != nil {
case common.ArtifactFormatGzip: _ = pr.CloseWithError(err)
pr, pw := io.Pipe() return filename, nil, err
go c.generateGzipStream(pw) }
return name + ".gz", pr, nil
case common.ArtifactFormatRaw:
file, err := c.openRawStream()
return name, file, err go func() {
err := archiver.Archive(context.Background(), c.files)
_ = pw.CloseWithError(err)
}()
default: return filename, pr, nil
return "", nil, fmt.Errorf("unsupported archive format: %s", c.Format)
}
} }
func (c *ArtifactsUploaderCommand) Run() error { func (c *ArtifactsUploaderCommand) Run() error {
......
package helpers package helpers
import ( import (
"context"
"io/ioutil"
"net/http" "net/http"
"os" "os"
"path/filepath" "path/filepath"
...@@ -10,8 +12,8 @@ import ( ...@@ -10,8 +12,8 @@ import (
"github.com/sirupsen/logrus" "github.com/sirupsen/logrus"
"github.com/urfave/cli" "github.com/urfave/cli"
"gitlab.com/gitlab-org/gitlab-runner/commands/helpers/archive"
"gitlab.com/gitlab-org/gitlab-runner/common" "gitlab.com/gitlab-org/gitlab-runner/common"
"gitlab.com/gitlab-org/gitlab-runner/helpers/archives"
url_helpers "gitlab.com/gitlab-org/gitlab-runner/helpers/url" url_helpers "gitlab.com/gitlab-org/gitlab-runner/helpers/url"
"gitlab.com/gitlab-org/gitlab-runner/log" "gitlab.com/gitlab-org/gitlab-runner/log"
) )
...@@ -66,6 +68,40 @@ func (c *CacheArchiverCommand) upload(_ int) error { ...@@ -66,6 +68,40 @@ func (c *CacheArchiverCommand) upload(_ int) error {
return retryOnServerError(resp) return retryOnServerError(resp)
} }
func (c *CacheArchiverCommand) createZipFile(filename string) error {
err := os.MkdirAll(filepath.Dir(filename), 0700)
if err != nil {
return err
}
f, err := ioutil.TempFile(filepath.Dir(filename), "archive_")
if err != nil {
return err
}
defer os.Remove(f.Name())
defer f.Close()
logrus.Debugln("Temporary file:", f.Name())
archiver, err := archive.NewArchiver(archive.Zip, f, c.wd, archive.DefaultCompression)
if err != nil {
return err
}
// Create archive
err = archiver.Archive(context.Background(), c.files)
if err != nil {
return err
}
err = f.Close()
if err != nil {
return err
}
return os.Rename(f.Name(), filename)
}
func (c *CacheArchiverCommand) Execute(*cli.Context) { func (c *CacheArchiverCommand) Execute(*cli.Context) {
log.SetRunnerFormatter() log.SetRunnerFormatter()
...@@ -87,7 +123,7 @@ func (c *CacheArchiverCommand) Execute(*cli.Context) { ...@@ -87,7 +123,7 @@ func (c *CacheArchiverCommand) Execute(*cli.Context) {
} }
// Create archive // Create archive
err = archives.CreateZipFile(c.File, c.sortedFiles()) err = c.createZipFile(c.File)
if err != nil { if err != nil {
logrus.Fatalln(err) logrus.Fatalln(err)
} }
......
package helpers package helpers
import ( import (
"context"
"io" "io"
"io/ioutil" "io/ioutil"
"net/http" "net/http"
...@@ -11,8 +12,8 @@ import ( ...@@ -11,8 +12,8 @@ import (
"github.com/sirupsen/logrus" "github.com/sirupsen/logrus"
"github.com/urfave/cli" "github.com/urfave/cli"
"gitlab.com/gitlab-org/gitlab-runner/commands/helpers/archive"
"gitlab.com/gitlab-org/gitlab-runner/common" "gitlab.com/gitlab-org/gitlab-runner/common"
"gitlab.com/gitlab-org/gitlab-runner/helpers/archives"
url_helpers "gitlab.com/gitlab-org/gitlab-runner/helpers/url" url_helpers "gitlab.com/gitlab-org/gitlab-runner/helpers/url"
"gitlab.com/gitlab-org/gitlab-runner/log" "gitlab.com/gitlab-org/gitlab-runner/log"
) )
...@@ -105,9 +106,14 @@ func (c *CacheExtractorCommand) getCache() (*http.Response, error) { ...@@ -105,9 +106,14 @@ func (c *CacheExtractorCommand) getCache() (*http.Response, error) {
return resp, retryOnServerError(resp) return resp, retryOnServerError(resp)
} }
func (c *CacheExtractorCommand) Execute(context *cli.Context) { func (c *CacheExtractorCommand) Execute(cliContext *cli.Context) {
log.SetRunnerFormatter() log.SetRunnerFormatter()
wd, err := os.Getwd()
if err != nil {
logrus.Fatalln("Unable to get working directory")
}
if c.File == "" { if c.File == "" {
logrus.Fatalln("Missing cache file") logrus.Fatalln("Missing cache file")
} }
...@@ -123,8 +129,22 @@ func (c *CacheExtractorCommand) Execute(context *cli.Context) { ...@@ -123,8 +129,22 @@ func (c *CacheExtractorCommand) Execute(context *cli.Context) {
"Instead a local version of cache will be extracted.") "Instead a local version of cache will be extracted.")
} }
err := archives.ExtractZipFile(c.File) f, size, err := openZip(c.File)
if err != nil && !os.IsNotExist(err) { if os.IsNotExist(err) {
return
}
if err != nil {
logrus.Fatalln(err)
}
defer f.Close()
extractor, err := archive.NewExtractor(archive.Zip, f, size, wd)
if err != nil {
logrus.Fatalln(err)
}
err = extractor.Extract(context.Background())
if err != nil {
logrus.Fatalln(err) logrus.Fatalln(err)
} }
} }
......
...@@ -3,9 +3,7 @@ package archives ...@@ -3,9 +3,7 @@ package archives
import ( import (
"archive/zip" "archive/zip"
"io" "io"
"io/ioutil"
"os" "os"
"path/filepath"
"github.com/sirupsen/logrus" "github.com/sirupsen/logrus"
) )
...@@ -103,34 +101,3 @@ func CreateZipArchive(w io.Writer, fileNames []string) error { ...@@ -103,34 +101,3 @@ func CreateZipArchive(w io.Writer, fileNames []string) error {
return nil return nil
} }
func CreateZipFile(fileName string, fileNames []string) error {
// create directories to store archive
err := os.MkdirAll(filepath.Dir(fileName), 0700)
if err != nil {
return err
}
tempFile, err := ioutil.TempFile(filepath.Dir(fileName), "archive_")
if err != nil {
return err
}
defer func() {
_ = tempFile.Close()
_ = os.Remove(tempFile.Name())
}()
logrus.Debugln("Temporary file:", tempFile.Name())
err = CreateZipArchive(tempFile, fileNames)
if err != nil {
return err
}
_ = tempFile.Close()
err = os.Rename(tempFile.Name(), fileName)
if err != nil {
return err
}
return nil
}
...@@ -116,8 +116,14 @@ func TestZipCreate(t *testing.T) { ...@@ -116,8 +116,14 @@ func TestZipCreate(t *testing.T) {
createTestPipe(t, multiBytes), createTestPipe(t, multiBytes),
"non_existing_file.txt", "non_existing_file.txt",
} }
err := CreateZipFile(fileName, paths)
f, err := os.Create(fileName)
require.NoError(t, err)
defer f.Close()
err = CreateZipArchive(f, paths)
require.NoError(t, err) require.NoError(t, err)
require.NoError(t, f.Close())
archive, err := zip.OpenReader(fileName) archive, err := zip.OpenReader(fileName)
require.NoError(t, err) require.NoError(t, err)
...@@ -158,8 +164,14 @@ func TestZipCreateWithGitPath(t *testing.T) { ...@@ -158,8 +164,14 @@ func TestZipCreateWithGitPath(t *testing.T) {
createTestGitPathFile(t, singleByte), createTestGitPathFile(t, singleByte),
createTestGitPathFile(t, multiBytes), createTestGitPathFile(t, multiBytes),
} }
err := CreateZipFile(fileName, paths)
f, err := os.Create(fileName)
require.NoError(t, err)
defer f.Close()
err = CreateZipArchive(f, paths)
require.NoError(t, err) require.NoError(t, err)
require.NoError(t, f.Close())
assert.Contains(t, buf.String(), "Part of .git directory is on the list of files to archive") assert.Contains(t, buf.String(), "Part of .git directory is on the list of files to archive")
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment