store-only on config and cli. use builder's dependencies

.travis.yml

@@ -17,4 +17,6 @@ install:
 
 script:
   - ./gomake clean build test quality install
+  - mkdir -p ~/.config/dgr
+  - 'echo -e "rkt:\n pullPolicy: new" > ~/.config/dgr/config.yml'
   - sudo PATH=${GOPATH}/bin:$PATH ./examples/build.sh

aci-builder/bin-run/builder.go

@@ -124,12 +124,32 @@ func (b *Builder) writeManifest() error {
 		aciManifest.NameAndVersion = *common.NewACFullnameWithVersion(aciManifest.NameAndVersion, common.GenerateVersion(b.aciTargetPath))
 	}
 
+	if err := b.fillDependenciesFromBuilder(aciManifest); err != nil {
+		return err
+	}
+
 	if err := common.WriteAciManifest(aciManifest, target, aciManifest.NameAndVersion.Name(), dgrVersion); err != nil {
 		return errs.WithEF(err, b.fields.WithField("file", target), "Failed to write manifest")
 	}
 	return nil
 }
 
+func (b *Builder) fillDependenciesFromBuilder(manifest *common.AciManifest) error {
+	manifestPath := b.aciTargetPath + common.PathBuilder + common.PathManifest
+	source, err := ioutil.ReadFile(manifestPath)
+	if err != nil {
+		return errs.WithEF(err, b.fields.WithField("path", manifestPath), "Failed to read builder manifest")
+	}
+
+	im := &schema.ImageManifest{}
+	if err = im.UnmarshalJSON(source); err != nil {
+		return errs.WithEF(err, b.fields.WithField("content", string(source)), "Cannot unmarshall json content")
+	}
+
+	(*manifest).Aci.Dependencies = common.FromAppcDependencies(im.Dependencies)
+	return nil
+}
+
 func (b *Builder) tarAci() error {
 	upperId, err := b.upperTreeStoreId()
 	if err != nil {

dgr/aci-build.go

@@ -9,7 +9,6 @@ import (
 	"github.com/appc/spec/schema"
 	"github.com/appc/spec/schema/types"
 	"github.com/blablacar/dgr/dgr/common"
-	"github.com/ghodss/yaml"
 	"github.com/n0rad/go-erlog/errs"
 	"github.com/n0rad/go-erlog/logs"
 )
@@ -56,12 +55,7 @@ func (aci *Aci) RunBuilderCommand(command common.BuilderCommand) error {
 		return errs.WithEF(err, aci.fields, "Cannot create target directory")
 	}
 
-	manifestMarshaled, err := yaml.Marshal(aci.manifest)
-	if err != nil {
-		return errs.WithEF(err, aci.fields, "Failed to marshal dgr manifest")
-	}
-
-	if err := ioutil.WriteFile(aci.target+common.PathManifestYmlTmpl, manifestMarshaled, 0644); err != nil {
+	if err := ioutil.WriteFile(aci.target+common.PathManifestYmlTmpl, []byte(aci.manifestTmpl), 0644); err != nil {
 		return errs.WithEF(err, aci.fields.WithField("file", aci.target+common.PathManifestYmlTmpl), "Failed to write manifest template")
 	}
 
@@ -143,7 +137,7 @@ func (aci *Aci) prepareStage1aci() (string, error) {
 	logs.WithFields(aci.fields).Debug("Preparing stage1")
 
 	if err := os.MkdirAll(aci.target+pathStage1+common.PathRootfs, 0777); err != nil {
-		return "", errs.WithEF(err, aci.fields.WithField("path", aci.target+pathBuilder), "Failed to create stage1 aci path")
+		return "", errs.WithEF(err, aci.fields.WithField("path", aci.target+common.PathBuilder), "Failed to create stage1 aci path")
 	}
 
 	Home.Rkt.Fetch(aci.manifest.Builder.Image.String(), common.PullPolicyNew)
@@ -197,24 +191,24 @@ func (aci *Aci) prepareStage1aci() (string, error) {
 func (aci *Aci) prepareBuildAci() (string, error) {
 	logs.WithFields(aci.fields).Debug("Preparing builder")
 
-	if err := os.MkdirAll(aci.target+pathBuilder+common.PathRootfs, 0777); err != nil {
-		return "", errs.WithEF(err, aci.fields.WithField("path", aci.target+pathBuilder), "Failed to create builder aci path")
+	if err := os.MkdirAll(aci.target+common.PathBuilder+common.PathRootfs, 0777); err != nil {
+		return "", errs.WithEF(err, aci.fields.WithField("path", aci.target+common.PathBuilder), "Failed to create builder aci path")
 	}
 
-	if err := ioutil.WriteFile(aci.target+pathBuilder+common.PathRootfs+"/.keep", []byte(""), 0644); err != nil {
-		return "", errs.WithEF(err, aci.fields.WithField("file", aci.target+pathBuilder+common.PathRootfs+"/.keep"), "Failed to write keep file")
+	if err := ioutil.WriteFile(aci.target+common.PathBuilder+common.PathRootfs+"/.keep", []byte(""), 0644); err != nil {
+		return "", errs.WithEF(err, aci.fields.WithField("file", aci.target+common.PathBuilder+common.PathRootfs+"/.keep"), "Failed to write keep file")
 	}
 	aci.manifest.Aci.App.Isolators = []common.Isolator{{Name: "os/linux/capabilities-retain-set", Value: common.LinuxCapabilitiesSetValue{Set: []types.LinuxCapability{"all"}}}}
 
-	if err := common.WriteAciManifest(aci.manifest, aci.target+pathBuilder+common.PathManifest, common.PrefixBuilder+aci.manifest.NameAndVersion.Name(), BuildVersion); err != nil {
+	if err := common.WriteAciManifest(aci.manifest, aci.target+common.PathBuilder+common.PathManifest, common.PrefixBuilder+aci.manifest.NameAndVersion.Name(), BuildVersion); err != nil {
 		return "", err
 	}
-	if err := aci.tarAci(aci.target + pathBuilder); err != nil {
+	if err := aci.tarAci(aci.target + common.PathBuilder); err != nil {
 		return "", err
 	}
 
-	logs.WithF(aci.fields.WithField("path", aci.target+pathBuilder+pathImageAci)).Info("Importing build to rkt")
-	hash, err := Home.Rkt.FetchInsecure(aci.target + pathBuilder + pathImageAci)
+	logs.WithF(aci.fields.WithField("path", aci.target+common.PathBuilder+pathImageAci)).Info("Importing build to rkt")
+	hash, err := Home.Rkt.FetchInsecure(aci.target + common.PathBuilder + pathImageAci)
 	if err != nil {
 		return "", errs.WithEF(err, aci.fields, "fetch of builder aci failed")
 	}

dgr/aci.go

@@ -25,7 +25,6 @@ const pathManifestJson = "/manifest.json"
 
 const pathStage1 = "/stage1"
 
-const pathBuilder = "/builder"
 const pathBuilderUuid = "/builder.uuid"
 const pathTesterUuid = "/tester.uuid"
 const pathVersion = "/version"
@@ -41,6 +40,7 @@ type Aci struct {
 	target          string
 	podName         *common.ACFullname
 	manifest        *common.AciManifest
+	manifestTmpl    string
 	args            BuildArgs
 	FullyResolveDep bool
 }
@@ -76,6 +76,7 @@ func NewAciWithManifest(path string, args BuildArgs, manifestTmpl string) (*Aci,
 		args:            args,
 		path:            fullPath,
 		manifest:        manifest,
+		manifestTmpl:    manifestTmpl,
 		target:          target,
 		FullyResolveDep: true,
 	}

dgr/common/aci-manifest.go

@@ -157,6 +157,15 @@ func ToAppcDependencies(dependencies []ACFullname) (types.Dependencies, error) {
 	return appcDependencies, nil
 }
 
+func FromAppcDependencies(dependencies types.Dependencies) []ACFullname {
+	deps := []ACFullname{}
+	for _, dep := range dependencies {
+		v, _ := dep.Labels.Get("version")
+		deps = append(deps, *NewACFullnameFromNameAndVersion(string(dep.ImageName), v))
+	}
+	return deps
+}
+
 func ToAppcIsolators(isos []Isolator) (types.Isolators, error) {
 	isolators := types.Isolators{}
 	for _, i := range isos {

dgr/common/common.go

@@ -5,6 +5,7 @@ import (
 	"github.com/n0rad/go-erlog/errs"
 )
 
+const PathBuilder = "/builder"
 const PathImageAci = "/image.aci"
 const PathManifest = "/manifest"
 const PathRootfs = "/rootfs"

dgr/common/rkt-client.go

@@ -48,6 +48,7 @@ type RktConfig struct {
 	LocalConfig        string        `yaml:"localConfig"`
 	SystemConfig       string        `yaml:"systemConfig"`
 	UserConfig         string        `yaml:"userConfig"`
+	PullPolicy         PullPolicy    `yaml:"pullPolicy"`
 	TrustKeysFromHttps bool          `yaml:"trustKeysFromHttps"`
 	NoStore            bool          `yaml:"noStore"`
 	StoreOnly          bool          `yaml:"storeOnly"`
@@ -160,12 +161,27 @@ func (rkt *RktClient) GetVersion() (Version, error) {
 type PullPolicy string
 
 const (
+	PullPolicyNone   PullPolicy = "none"
 	PullPolicyNew    PullPolicy = "new"
 	PullPolicyUpdate PullPolicy = "update"
 )
 
+func (p PullPolicy) IsValid() bool {
+	switch p {
+	case PullPolicyNew, PullPolicyNone, PullPolicyUpdate:
+	default:
+		return false
+	}
+	return true
+}
+
 func (rkt *RktClient) Fetch(image string, pullPolicy PullPolicy) (string, error) {
 	globalArgs := rkt.globalArgs
+
+	if rkt.config.PullPolicy != "" {
+		pullPolicy = rkt.config.PullPolicy
+	}
+
 	if rkt.Version.GreaterThanOrEqualTo(rktVersionWithPullPolicy) {
 		globalArgs = append(globalArgs, "--pull-policy="+string(pullPolicy))
 	}

dgr/common/version_test.go

@@ -32,4 +32,6 @@ func TestCompareVersion(t *testing.T) {
 	assertVersion(t, "2.21-2-2", "2.21-2-2", 0)
 	assertVersion(t, "2.21-2-2", "2.21-2-1", 1)
 	assertVersion(t, "2.21-2a", "2.21-2b", -1)
+	assertVersion(t, "2.21-2a", "2.21-2bccccc", -1)
+	assertVersion(t, "2.21-2accccccc", "2.21-2b", -1)
 }

dgr/home.go

@@ -74,11 +74,8 @@ func NewHome(path string) HomeStruct {
 		}
 	}
 
-	if Args.NoStore {
-		config.Rkt.NoStore = true
-	}
-	if Args.StoreOnly {
-		config.Rkt.StoreOnly = true
+	if Args.PullPolicy != "" {
+		config.Rkt.PullPolicy = common.PullPolicy(Args.PullPolicy)
 	}
 	if config.Signs == nil {
 		config.Signs = &[]Sign{{Disabled: true}}

dgr/main.go

@@ -24,15 +24,14 @@ var Args = BuildArgs{}
 var workPath string
 
 type BuildArgs struct {
-	NoStore       bool
-	StoreOnly     bool
 	Force         bool
 	Test          bool
 	NoTestFail    bool
 	KeepBuilder   bool
 	CatchOnError  bool
 	CatchOnStep   bool
 	ParallelBuild bool
+	PullPolicy    string
 	SetEnv        envMap
 }
 
@@ -80,18 +79,20 @@ func Execute() {
 			if targetRootPath != "" {
 				Home.Config.TargetWorkDir = targetRootPath
 			}
+
+			if Args.PullPolicy != "" && !common.PullPolicy(Args.PullPolicy).IsValid() {
+				logs.WithField("pull-policy", Args.PullPolicy).Fatal("Invalid pull-policy")
+			}
 		},
 	}
-	//rootCmd.PersistentFlags().BoolVarP(&Args.Clean, "clean", "c", false, "Clean before doing anything")
 	rootCmd.PersistentFlags().StringVarP(&targetRootPath, "targets-root-path", "p", "", "Set targets root path")
 	rootCmd.PersistentFlags().StringVarP(&logLevel, "loglevel", "L", "info", "Set log level")
 	rootCmd.PersistentFlags().StringVar(&logLevel, "log-level", "info", "Set log level")
 	rootCmd.PersistentFlags().StringVarP(&homePath, "home-path", "H", DefaultHomeFolder(""), "Set home folder")
 	rootCmd.PersistentFlags().StringVarP(&workPath, "work-path", "W", ".", "Set the work path")
 	rootCmd.PersistentFlags().BoolVarP(&version, "version", "V", false, "Display dgr version")
 	rootCmd.PersistentFlags().Var(&Args.SetEnv, "set-env", "Env passed to builder scripts")
-	rootCmd.PersistentFlags().BoolVar(&Args.StoreOnly, "store-only", false, "Tell rkt to use the store only")
-	rootCmd.PersistentFlags().BoolVar(&Args.NoStore, "no-store", false, "Tell rkt to not use store")
+	rootCmd.PersistentFlags().StringVar(&Args.PullPolicy, "pull-policy", "", "force rkt fetch Policy")
 	rootCmd.PersistentFlags().BoolVarP(&Args.ParallelBuild, "parallel", "P", false, "Run build in parallel for pod")
 
 	rootCmd.AddCommand(buildCmd, cleanCmd, pushCmd, installCmd, testCmd, versionCmd, initCmd, graphCmd, tryCmd, signCmd, aciVersion, configCmd, updateCmd)

dgr/pod-build.go

@@ -211,7 +211,7 @@ func (p *Pod) buildAci(e common.RuntimeApp) (*Aci, error) {
 
 	// TODO attributes should be builder dependent only
 	if empty, err := common.IsDirEmpty(p.path + "/attributes"); !empty && err == nil {
-		path := aci.target + pathBuilder + common.PathRootfs + "/dgr/pod/attributes"
+		path := aci.target + common.PathBuilder + common.PathRootfs + "/dgr/pod/attributes"
 		if err := os.MkdirAll(path, 0777); err != nil {
 			return nil, errs.WithEF(err, aci.fields.WithField("path", path), "Failed to create pod attributes directory in builder")
 		}

examples/aci-base/aci-manifest.yml

@@ -1 +1 @@
-name: blablacar.github.io/dgr/aci-base
+name: blablacar.github.io/dgr/aci-base:1

examples/none/aci-grafana/aci-manifest.yml

@@ -1,7 +1,7 @@
 name: blablacar.github.io/dgr/aci-grafana:2.6.0-1
 aci:
   dependencies:
-    - blablacar.github.io/dgr/aci-libc
+    - blablacar.github.io/dgr/aci-libc:1
   app:
     exec: [ /grafana/bin/grafana-server ]
     workingDirectory: /grafana/bin

examples/none/aci-libc/aci-manifest.yml

@@ -1,4 +1,4 @@
-name: blablacar.github.io/dgr/aci-libc
+name: blablacar.github.io/dgr/aci-libc:1
 aci:
   dependencies:
-    - blablacar.github.io/dgr/aci-base
+    - blablacar.github.io/dgr/aci-base:1

examples/none/aci-prometheus/aci-manifest.yml

@@ -1,7 +1,7 @@
 name: blablacar.github.io/dgr/aci-prometheus:0.16.1-1nerve2
 aci:
   dependencies:
-    - blablacar.github.io/dgr/aci-libc
+    - blablacar.github.io/dgr/aci-libc:1
   app:
     exec: [/prometheus/prometheus, -config.file, /etc/prometheus/prometheus.yml]
     mountPoints:

examples/none/aci-rkt/aci-manifest.yml

@@ -1,4 +1,4 @@
 name: blablacar.github.io/dgr/aci-rkt:1.3.0
 aci:
   dependencies:
-    - blablacar.github.io/dgr/aci-libc
+    - blablacar.github.io/dgr/aci-libc:1