Unverified Commit c5bc5753 authored by Tomasz Maczukin's avatar Tomasz Maczukin Committed by Tomasz Maczukin

Merge branch 'make-volumes-to-work-on-linux-docker-on-windows' into 'master'

Make volumes to work on linux docker on windows

Closes #4251

See merge request gitlab-org/gitlab-runner!1363
parent a419fbe4
...@@ -13,3 +13,5 @@ const waitForContainerTimeout = 15 * time.Second ...@@ -13,3 +13,5 @@ const waitForContainerTimeout = 15 * time.Second
const osTypeLinux = "linux" const osTypeLinux = "linux"
const osTypeWindows = "windows" const osTypeWindows = "windows"
const metadataOSType = "OSType"
...@@ -52,8 +52,9 @@ var errVolumesManagerUndefined = errors.New("volumesManager is undefined") ...@@ -52,8 +52,9 @@ var errVolumesManagerUndefined = errors.New("volumesManager is undefined")
type executor struct { type executor struct {
executors.AbstractExecutor executors.AbstractExecutor
client docker_helpers.Client client docker_helpers.Client
info types.Info volumeParser parser.Parser
info types.Info
temporary []string // IDs of containers that should be removed temporary []string // IDs of containers that should be removed
...@@ -987,8 +988,8 @@ func (e *executor) connectDocker() error { ...@@ -987,8 +988,8 @@ func (e *executor) connectDocker() error {
// validateOSType checks if the ExecutorOptions metadata matches with the docker // validateOSType checks if the ExecutorOptions metadata matches with the docker
// info response. // info response.
func (e *executor) validateOSType() error { func (e *executor) validateOSType() error {
executorOSType, ok := e.ExecutorOptions.Metadata["OSType"] executorOSType := e.ExecutorOptions.Metadata[metadataOSType]
if !ok { if executorOSType == "" {
return common.MakeBuildError("%s does not have any OSType specified", e.Config.Executor) return common.MakeBuildError("%s does not have any OSType specified", e.Config.Executor)
} }
...@@ -1142,21 +1143,14 @@ func (e *executor) Prepare(options common.ExecutorPrepareOptions) error { ...@@ -1142,21 +1143,14 @@ func (e *executor) Prepare(options common.ExecutorPrepareOptions) error {
return nil return nil
} }
var (
buildDirectoryNotAbsoluteErr = common.MakeBuildError("build directory needs to be an absolute path")
buildDirectoryIsRootPathErr = common.MakeBuildError("build directory needs to be a non-root path")
)
func (e *executor) prepareBuildsDir(options common.ExecutorPrepareOptions) error { func (e *executor) prepareBuildsDir(options common.ExecutorPrepareOptions) error {
volumeParser, err := parser.New(e.info) if e.volumeParser == nil {
if err != nil { return common.MakeBuildError("missing volume parser")
return fmt.Errorf("couldn't create volumes parser: %v", err)
} }
isHostMounted, err := volumes.IsHostMountedVolume(volumeParser, e.RootDir(), options.Config.Docker.Volumes...) isHostMounted, err := volumes.IsHostMountedVolume(e.volumeParser, e.RootDir(), options.Config.Docker.Volumes...)
if err != nil { if err != nil {
return err return &common.BuildError{Inner: err}
} }
// We need to set proper value for e.SharedBuildsDir because // We need to set proper value for e.SharedBuildsDir because
...@@ -1168,14 +1162,6 @@ func (e *executor) prepareBuildsDir(options common.ExecutorPrepareOptions) error ...@@ -1168,14 +1162,6 @@ func (e *executor) prepareBuildsDir(options common.ExecutorPrepareOptions) error
e.SharedBuildsDir = true e.SharedBuildsDir = true
} }
if !filepath.IsAbs(e.RootDir()) {
return buildDirectoryNotAbsoluteErr
}
if e.RootDir() == "/" {
return buildDirectoryIsRootPathErr
}
return nil return nil
} }
......
...@@ -9,6 +9,7 @@ import ( ...@@ -9,6 +9,7 @@ import (
"gitlab.com/gitlab-org/gitlab-runner/common" "gitlab.com/gitlab-org/gitlab-runner/common"
"gitlab.com/gitlab-org/gitlab-runner/executors" "gitlab.com/gitlab-org/gitlab-runner/executors"
"gitlab.com/gitlab-org/gitlab-runner/executors/docker/internal/volumes/parser"
) )
type commandExecutor struct { type commandExecutor struct {
...@@ -115,7 +116,7 @@ func init() { ...@@ -115,7 +116,7 @@ func init() {
}, },
ShowHostname: true, ShowHostname: true,
Metadata: map[string]string{ Metadata: map[string]string{
"OSType": osTypeLinux, metadataOSType: osTypeLinux,
}, },
} }
...@@ -125,6 +126,7 @@ func init() { ...@@ -125,6 +126,7 @@ func init() {
AbstractExecutor: executors.AbstractExecutor{ AbstractExecutor: executors.AbstractExecutor{
ExecutorOptions: options, ExecutorOptions: options,
}, },
volumeParser: parser.NewLinuxParser(),
}, },
} }
e.SetCurrentStage(common.ExecutorStageCreated) e.SetCurrentStage(common.ExecutorStageCreated)
......
...@@ -3,6 +3,7 @@ package docker ...@@ -3,6 +3,7 @@ package docker
import ( import (
"gitlab.com/gitlab-org/gitlab-runner/common" "gitlab.com/gitlab-org/gitlab-runner/common"
"gitlab.com/gitlab-org/gitlab-runner/executors" "gitlab.com/gitlab-org/gitlab-runner/executors"
"gitlab.com/gitlab-org/gitlab-runner/executors/docker/internal/volumes/parser"
) )
func init() { func init() {
...@@ -18,7 +19,7 @@ func init() { ...@@ -18,7 +19,7 @@ func init() {
}, },
ShowHostname: true, ShowHostname: true,
Metadata: map[string]string{ Metadata: map[string]string{
"OSType": osTypeWindows, metadataOSType: osTypeWindows,
}, },
} }
...@@ -28,6 +29,7 @@ func init() { ...@@ -28,6 +29,7 @@ func init() {
AbstractExecutor: executors.AbstractExecutor{ AbstractExecutor: executors.AbstractExecutor{
ExecutorOptions: options, ExecutorOptions: options,
}, },
volumeParser: parser.NewWindowsParser(),
}, },
} }
e.SetCurrentStage(common.ExecutorStageCreated) e.SetCurrentStage(common.ExecutorStageCreated)
......
...@@ -7,6 +7,7 @@ import ( ...@@ -7,6 +7,7 @@ import (
"gitlab.com/gitlab-org/gitlab-runner/common" "gitlab.com/gitlab-org/gitlab-runner/common"
"gitlab.com/gitlab-org/gitlab-runner/executors" "gitlab.com/gitlab-org/gitlab-runner/executors"
"gitlab.com/gitlab-org/gitlab-runner/executors/docker/internal/volumes/parser"
"gitlab.com/gitlab-org/gitlab-runner/helpers/ssh" "gitlab.com/gitlab-org/gitlab-runner/helpers/ssh"
) )
...@@ -93,7 +94,7 @@ func init() { ...@@ -93,7 +94,7 @@ func init() {
}, },
ShowHostname: true, ShowHostname: true,
Metadata: map[string]string{ Metadata: map[string]string{
"OSType": osTypeLinux, metadataOSType: osTypeLinux,
}, },
} }
...@@ -103,6 +104,7 @@ func init() { ...@@ -103,6 +104,7 @@ func init() {
AbstractExecutor: executors.AbstractExecutor{ AbstractExecutor: executors.AbstractExecutor{
ExecutorOptions: options, ExecutorOptions: options,
}, },
volumeParser: parser.NewLinuxParser(),
}, },
} }
e.SetCurrentStage(common.ExecutorStageCreated) e.SetCurrentStage(common.ExecutorStageCreated)
......
...@@ -205,6 +205,7 @@ func testServiceFromNamedImage(t *testing.T, description, imageName, serviceName ...@@ -205,6 +205,7 @@ func testServiceFromNamedImage(t *testing.T, description, imageName, serviceName
OSType: helperimage.OSTypeLinux, OSType: helperimage.OSTypeLinux,
Architecture: "amd64", Architecture: "amd64",
}, },
volumeParser: parser.NewLinuxParser(),
} }
options := buildImagePullOptions(e, imageName) options := buildImagePullOptions(e, imageName)
...@@ -567,54 +568,54 @@ func TestDockerGetExistingDockerImageIfPullFails(t *testing.T) { ...@@ -567,54 +568,54 @@ func TestDockerGetExistingDockerImageIfPullFails(t *testing.T) {
func TestPrepareBuildsDir(t *testing.T) { func TestPrepareBuildsDir(t *testing.T) {
tests := map[string]struct { tests := map[string]struct {
osType string parser parser.Parser
rootDir string rootDir string
volumes []string volumes []string
expectedSharedBuildsDir bool expectedSharedBuildsDir bool
expectedError error expectedError string
}{ }{
"rootDir mounted as host based volume": { "rootDir mounted as host based volume": {
osType: parser.OSTypeLinux, parser: parser.NewLinuxParser(),
rootDir: "/build", rootDir: "/build",
volumes: []string{"/build:/build"}, volumes: []string{"/build:/build"},
expectedSharedBuildsDir: true, expectedSharedBuildsDir: true,
}, },
"rootDir mounted as container based volume": { "rootDir mounted as container based volume": {
osType: parser.OSTypeLinux, parser: parser.NewLinuxParser(),
rootDir: "/build", rootDir: "/build",
volumes: []string{"/build"}, volumes: []string{"/build"},
expectedSharedBuildsDir: false, expectedSharedBuildsDir: false,
}, },
"rootDir not mounted as volume": { "rootDir not mounted as volume": {
osType: parser.OSTypeLinux, parser: parser.NewLinuxParser(),
rootDir: "/build", rootDir: "/build",
volumes: []string{"/folder:/folder"}, volumes: []string{"/folder:/folder"},
expectedSharedBuildsDir: false, expectedSharedBuildsDir: false,
}, },
"rootDir's parent mounted as volume": { "rootDir's parent mounted as volume": {
osType: parser.OSTypeLinux, parser: parser.NewLinuxParser(),
rootDir: "/build/other/directory", rootDir: "/build/other/directory",
volumes: []string{"/build/:/build"}, volumes: []string{"/build/:/build"},
expectedSharedBuildsDir: true, expectedSharedBuildsDir: true,
}, },
"rootDir is not an absolute path": { "rootDir is not an absolute path": {
osType: parser.OSTypeLinux, parser: parser.NewLinuxParser(),
rootDir: "builds", rootDir: "builds",
expectedError: buildDirectoryNotAbsoluteErr, expectedError: "build directory needs to be an absolute path",
}, },
"rootDir is /": { "rootDir is /": {
osType: parser.OSTypeLinux, parser: parser.NewLinuxParser(),
rootDir: "/", rootDir: "/",
expectedError: buildDirectoryIsRootPathErr, expectedError: "build directory needs to be a non-root path",
}, },
"error on volume parsing": { "error on volume parsing": {
osType: parser.OSTypeLinux, parser: parser.NewLinuxParser(),
rootDir: "/build", rootDir: "/build",
volumes: []string{""}, volumes: []string{""},
expectedError: parser.NewInvalidVolumeSpecErr(""), expectedError: "invalid volume specification",
}, },
"error on volume parser creation": { "error on volume parser creation": {
expectedError: errors.New(`couldn't create volumes parser: unsupported OSType ""`), expectedError: `missing volume parser`,
}, },
} }
...@@ -637,13 +638,17 @@ func TestPrepareBuildsDir(t *testing.T) { ...@@ -637,13 +638,17 @@ func TestPrepareBuildsDir(t *testing.T) {
AbstractExecutor: executors.AbstractExecutor{ AbstractExecutor: executors.AbstractExecutor{
Config: c, Config: c,
}, },
info: types.Info{ volumeParser: test.parser,
OSType: test.osType,
},
} }
err := e.prepareBuildsDir(options) err := e.prepareBuildsDir(options)
assert.Equal(t, test.expectedError, err) if test.expectedError != "" {
assert.Error(t, err)
assert.Contains(t, err.Error(), test.expectedError)
return
}
assert.NoError(t, err)
assert.Equal(t, test.expectedSharedBuildsDir, e.SharedBuildsDir) assert.Equal(t, test.expectedSharedBuildsDir, e.SharedBuildsDir)
}) })
} }
...@@ -1408,10 +1413,11 @@ func TestDockerWatchOn_1_12_4(t *testing.T) { ...@@ -1408,10 +1413,11 @@ func TestDockerWatchOn_1_12_4(t *testing.T) {
AbstractExecutor: executors.AbstractExecutor{ AbstractExecutor: executors.AbstractExecutor{
ExecutorOptions: executors.ExecutorOptions{ ExecutorOptions: executors.ExecutorOptions{
Metadata: map[string]string{ Metadata: map[string]string{
"OSType": "linux", metadataOSType: osTypeLinux,
}, },
}, },
}, },
volumeParser: parser.NewLinuxParser(),
} }
e.Context = context.Background() e.Context = context.Background()
e.Build = &common.Build{ e.Build = &common.Build{
...@@ -1486,6 +1492,7 @@ func prepareTestDockerConfiguration(t *testing.T, dockerConfig *common.DockerCon ...@@ -1486,6 +1492,7 @@ func prepareTestDockerConfiguration(t *testing.T, dockerConfig *common.DockerCon
e := &executor{} e := &executor{}
e.client = c e.client = c
e.volumeParser = parser.NewLinuxParser()
e.info = types.Info{ e.info = types.Info{
OSType: helperimage.OSTypeLinux, OSType: helperimage.OSTypeLinux,
Architecture: "amd64", Architecture: "amd64",
...@@ -1733,21 +1740,21 @@ func TestCheckOSType(t *testing.T) { ...@@ -1733,21 +1740,21 @@ func TestCheckOSType(t *testing.T) {
}{ }{
"executor and docker info mismatch": { "executor and docker info mismatch": {
executorMetadata: map[string]string{ executorMetadata: map[string]string{
"OSType": "windows", metadataOSType: osTypeWindows,
}, },
dockerInfoOSType: "linux", dockerInfoOSType: osTypeLinux,
expectedErr: "executor requires OSType=windows, but Docker Engine supports only OSType=linux", expectedErr: "executor requires OSType=windows, but Docker Engine supports only OSType=linux",
}, },
"executor and docker info match": { "executor and docker info match": {
executorMetadata: map[string]string{ executorMetadata: map[string]string{
"OSType": "linux", metadataOSType: osTypeLinux,
}, },
dockerInfoOSType: "linux", dockerInfoOSType: osTypeLinux,
expectedErr: "", expectedErr: "",
}, },
"executor OSType not defined": { "executor OSType not defined": {
executorMetadata: nil, executorMetadata: nil,
dockerInfoOSType: "linux", dockerInfoOSType: osTypeLinux,
expectedErr: " does not have any OSType specified", expectedErr: " does not have any OSType specified",
}, },
} }
......
...@@ -4,7 +4,6 @@ import ( ...@@ -4,7 +4,6 @@ import (
"context" "context"
"errors" "errors"
"fmt" "fmt"
"path/filepath"
"gitlab.com/gitlab-org/gitlab-runner/executors/docker/internal/volumes/parser" "gitlab.com/gitlab-org/gitlab-runner/executors/docker/internal/volumes/parser"
) )
...@@ -74,9 +73,14 @@ func (m *manager) Create(volume string) error { ...@@ -74,9 +73,14 @@ func (m *manager) Create(volume string) error {
} }
func (m *manager) addHostVolume(volume *parser.Volume) error { func (m *manager) addHostVolume(volume *parser.Volume) error {
volume.Destination = m.getAbsoluteContainerPath(volume.Destination) var err error
err := m.managedVolumes.Add(volume.Destination) volume.Destination, err = m.getAbsoluteContainerPath(volume.Destination)
if err != nil {
return err
}
err = m.managedVolumes.Add(volume.Destination)
if err != nil { if err != nil {
return err return err
} }
...@@ -86,16 +90,21 @@ func (m *manager) addHostVolume(volume *parser.Volume) error { ...@@ -86,16 +90,21 @@ func (m *manager) addHostVolume(volume *parser.Volume) error {
return nil return nil
} }
func (m *manager) getAbsoluteContainerPath(dir string) string { func (m *manager) getAbsoluteContainerPath(dir string) (string, error) {
if filepath.IsAbs(dir) { if m.parser.Path().IsRoot(dir) {
return dir return "", errDirectoryIsRootPath
}
if m.parser.Path().IsAbs(dir) {
return dir, nil
} }
return filepath.Join(m.config.BaseContainerPath, dir) return m.parser.Path().Join(m.config.BaseContainerPath, dir), nil
} }
func (m *manager) appendVolumeBind(volume *parser.Volume) { func (m *manager) appendVolumeBind(volume *parser.Volume) {
m.logger.Debugln(fmt.Sprintf("Using host-based %q for %q...", volume.Source, volume.Destination)) m.logger.Debugln(fmt.Sprintf("Using host-based %q for %q...", volume.Source, volume.Destination))
m.volumeBindings = append(m.volumeBindings, volume.Definition()) m.volumeBindings = append(m.volumeBindings, volume.Definition())
} }
...@@ -118,19 +127,20 @@ func (m *manager) addCacheVolume(volume *parser.Volume) error { ...@@ -118,19 +127,20 @@ func (m *manager) addCacheVolume(volume *parser.Volume) error {
} }
func (m *manager) createHostBasedCacheVolume(containerPath string) error { func (m *manager) createHostBasedCacheVolume(containerPath string) error {
containerPath = m.getAbsoluteContainerPath(containerPath) var err error
err := m.managedVolumes.Add(containerPath) containerPath, err = m.getAbsoluteContainerPath(containerPath)
if err != nil { if err != nil {
return err return err
} }
hostPath := filepath.Join(m.config.CacheDir, m.config.UniqueName, hashContainerPath(containerPath)) err = m.managedVolumes.Add(containerPath)
hostPath, err = filepath.Abs(hostPath)
if err != nil { if err != nil {
return err return err
} }
hostPath := m.parser.Path().Join(m.config.CacheDir, m.config.UniqueName, hashContainerPath(containerPath))
m.appendVolumeBind(&parser.Volume{ m.appendVolumeBind(&parser.Volume{
Source: hostPath, Source: hostPath,
Destination: containerPath, Destination: containerPath,
...@@ -140,9 +150,12 @@ func (m *manager) createHostBasedCacheVolume(containerPath string) error { ...@@ -140,9 +150,12 @@ func (m *manager) createHostBasedCacheVolume(containerPath string) error {
} }
func (m *manager) createContainerBasedCacheVolume(containerPath string) (string, error) { func (m *manager) createContainerBasedCacheVolume(containerPath string) (string, error) {
containerPath = m.getAbsoluteContainerPath(containerPath) containerPath, err := m.getAbsoluteContainerPath(containerPath)
if err != nil {
return "", err
}
err := m.managedVolumes.Add(containerPath) err = m.managedVolumes.Add(containerPath)
if err != nil { if err != nil {
return "", err return "", err
} }
......
...@@ -10,6 +10,7 @@ import ( ...@@ -10,6 +10,7 @@ import (
"github.com/stretchr/testify/require" "github.com/stretchr/testify/require"
"gitlab.com/gitlab-org/gitlab-runner/executors/docker/internal/volumes/parser" "gitlab.com/gitlab-org/gitlab-runner/executors/docker/internal/volumes/parser"
"gitlab.com/gitlab-org/gitlab-runner/helpers/path"
) )
func newDebugLoggerMock() *mockDebugLogger { func newDebugLoggerMock() *mockDebugLogger {
...@@ -51,8 +52,9 @@ func addCacheContainerManager(manager *manager) *MockCacheContainersManager { ...@@ -51,8 +52,9 @@ func addCacheContainerManager(manager *manager) *MockCacheContainersManager {
func addParser(manager *manager) *parser.MockParser { func addParser(manager *manager) *parser.MockParser {
parserMock := new(parser.MockParser) parserMock := new(parser.MockParser)
manager.parser = parserMock parserMock.On("Path").Return(path.NewUnixPath())
manager.parser = parserMock
return parserMock return parserMock
} }
...@@ -101,6 +103,12 @@ func TestDefaultManager_CreateUserVolumes_HostVolume(t *testing.T) { ...@@ -101,6 +103,12 @@ func TestDefaultManager_CreateUserVolumes_HostVolume(t *testing.T) {
parsedVolume: &parser.Volume{Source: "/host/new", Destination: "/my/path", Mode: "ro"}, parsedVolume: &parser.Volume{Source: "/host/new", Destination: "/my/path", Mode: "ro"},
expectedBinding: []string{"/host:/duplicated", "/host/new:/my/path:ro"}, expectedBinding: []string{"/host:/duplicated", "/host/new:/my/path:ro"},
}, },
"root volume specified": {
volume: "/host/new:/:ro",
parsedVolume: &parser.Volume{Source: "/host/new", Destination: "/", Mode: "ro"},
expectedBinding: []string{"/host:/duplicated"},
expectedError: errDirectoryIsRootPath,
},
} }
for testName, testCase := range testCases { for testName, testCase := range testCases {
...@@ -451,8 +459,9 @@ func TestDefaultManager_CreateUserVolumes_CacheVolume_ContainerBased_WithError(t ...@@ -451,8 +459,9 @@ func TestDefaultManager_CreateUserVolumes_CacheVolume_ContainerBased_WithError(t
func TestDefaultManager_CreateUserVolumes_ParserError(t *testing.T) { func TestDefaultManager_CreateUserVolumes_ParserError(t *testing.T) {