Fix various issues with burrow deploy and Deploy tests

Allow Deploy jobs to mark themselves as not suitable for parallel execution

Signed-off-by: Silas Davis <[email protected]>

Makefile

@@ -84,7 +84,7 @@ PROTO_GO_FILES_REAL = $(shell find . -type f -name '*.pb.go' -print)
 PROTO_TS_FILES = $(patsubst %.proto, %.pb.ts, $(PROTO_FILES))
 
 .PHONY: protobuf
-protobuf: $(PROTO_GO_FILES) ${PROTO_TS_FILES} fix
+protobuf: $(PROTO_GO_FILES) $(PROTO_TS_FILES) fix
 
 # Implicit compile rule for GRPC/proto files (note since pb.go files no longer generated
 # in same directory as proto file this just regenerates everything

acm/validator/cache.go

@@ -4,6 +4,7 @@ import (
 	"math/big"
 
 	"github.com/hyperledger/burrow/crypto"
+	"github.com/hyperledger/burrow/util"
 )
 
 // Cache is just a Ring with no memory
@@ -22,6 +23,7 @@ func (vc *Cache) Reset(backend Iterable) {
 }
 
 func (vc *Cache) Flush(output Writer, backend Iterable) error {
+	util.Debugf("Flushing validators...")
 	err := vc.Delta.IterateValidators(func(id crypto.Addressable, power *big.Int) error {
 		_, err := output.SetPower(id.GetPublicKey(), power)
 		return err

cmd/burrow/commands/tx.go

@@ -10,7 +10,6 @@ import (
 	"github.com/hyperledger/burrow/config/source"
 	"github.com/hyperledger/burrow/deploy/def"
 	"github.com/hyperledger/burrow/deploy/jobs"
-	"github.com/hyperledger/burrow/deploy/util"
 	"github.com/hyperledger/burrow/logging"
 	"github.com/hyperledger/burrow/txs/payload"
 	cli "github.com/jawher/mow.cli"
@@ -217,7 +216,7 @@ func makeTx(client *def.Client, tx payload.Payload) (string, error) {
 		return "", err
 	}
 
-	util.ReadTxSignAndBroadcast(txe, err, logger)
+	jobs.LogTxExecution(txe, logger)
 	if err != nil {
 		return "", err
 	}

deploy/def/client.go

@@ -359,7 +359,7 @@ func (c *Client) UpdateAccount(arg *GovArg, logger *logging.Logger) (*payload.Go
 	}
 	update := &spec.TemplateAccount{
 		Permissions: arg.Permissions,
-		Roles:       arg.Permissions,
+		Roles:       arg.Roles,
 	}
 	if arg.Address != "" {
 		addr, err := c.ParseAddress(arg.Address, logger)
@@ -369,7 +369,7 @@ func (c *Client) UpdateAccount(arg *GovArg, logger *logging.Logger) (*payload.Go
 		update.Address = &addr
 	}
 	if arg.PublicKey != "" {
-		pubKey, err := publicKeyFromString(arg.PublicKey)
+		pubKey, err := PublicKeyFromString(arg.PublicKey)
 		if err != nil {
 			return nil, fmt.Errorf("could not parse publicKey: %v", err)
 		}
@@ -427,7 +427,7 @@ func (c *Client) PublicKeyFromAddress(address *crypto.Address) (*crypto.PublicKe
 	return &pubKey, nil
 }
 
-func publicKeyFromString(publicKey string) (crypto.PublicKey, error) {
+func PublicKeyFromString(publicKey string) (crypto.PublicKey, error) {
 	bs, err := hex.DecodeString(publicKey)
 	if err != nil {
 		return crypto.PublicKey{}, fmt.Errorf("could not parse public key string %s as hex: %v", publicKey, err)

deploy/def/deploy.go

@@ -39,18 +39,3 @@ func (args *DeployArgs) Validate() error {
 	)
 }
 
-type Playbook struct {
-	Filename string
-	Account  string
-	Jobs     []*Job
-	Path     string `mapstructure:"-" json:"-" yaml:"-" toml:"-"`
-	BinPath  string `mapstructure:"-" json:"-" yaml:"-" toml:"-"`
-	// If we're in a proposal or meta job, reference our parent script
-	Parent *Playbook `mapstructure:"-" json:"-" yaml:"-" toml:"-"`
-}
-
-func (pkg *Playbook) Validate() error {
-	return validation.ValidateStruct(pkg,
-		validation.Field(&pkg.Jobs),
-	)
-}

deploy/def/playbook.go

@@ -0,0 +1,21 @@
+package def
+
+import "github.com/go-ozzo/ozzo-validation"
+
+type Playbook struct {
+	Filename string
+	Account  string
+	// Prevent this playbook from running at the same time as other playbooks
+	NoParallel bool `mapstructure:"no-parallel,omitempty" json:"no-parallel,omitempty" yaml:"no-parallel,omitempty" toml:"no-parallel,omitempty"`
+	Jobs       []*Job
+	Path       string `mapstructure:"-" json:"-" yaml:"-" toml:"-"`
+	BinPath    string `mapstructure:"-" json:"-" yaml:"-" toml:"-"`
+	// If we're in a proposal or meta job, reference our parent script
+	Parent *Playbook `mapstructure:"-" json:"-" yaml:"-" toml:"-"`
+}
+
+func (pkg *Playbook) Validate() error {
+	return validation.ValidateStruct(pkg,
+		validation.Field(&pkg.Jobs),
+	)
+}

deploy/jobs/job_manager.go

@@ -155,12 +155,12 @@ func doJobs(playbook *def.Playbook, args *def.DeployArgs, client *def.Client, lo
 			if err != nil {
 				return err
 			}
-			err = UpdateAccountJob(job.UpdateAccount, playbook.Account, tx, client, logger)
+			err = UpdateAccountJob(tx, client, logger)
 
 		// Util jobs
 		case *def.Account:
 			announce(job.Name, "Account", logger)
-			job.Result, err = SetAccountJob(job.Account, args, playbook, logger)
+			job.Result, err = SetAccountJob(job.Account, playbook, logger)
 		case *def.Set:
 			announce(job.Name, "Set", logger)
 			job.Result, err = SetValJob(job.Set, args, logger)
@@ -172,7 +172,7 @@ func doJobs(playbook *def.Playbook, args *def.DeployArgs, client *def.Client, lo
 			if err != nil {
 				return err
 			}
-			job.Result, err = SendJob(job.Send, tx, playbook.Account, client, logger)
+			job.Result, err = SendJob(tx, client, logger)
 			if err != nil {
 				return err
 			}
@@ -182,7 +182,7 @@ func doJobs(playbook *def.Playbook, args *def.DeployArgs, client *def.Client, lo
 			if err != nil {
 				return err
 			}
-			job.Result, err = BondJob(job.Bond, tx, playbook.Account, client, logger)
+			job.Result, err = BondJob(tx, client, logger)
 			if err != nil {
 				return err
 			}
@@ -192,7 +192,7 @@ func doJobs(playbook *def.Playbook, args *def.DeployArgs, client *def.Client, lo
 			if err != nil {
 				return err
 			}
-			job.Result, err = UnbondJob(job.Unbond, tx, playbook.Account, client, logger)
+			job.Result, err = UnbondJob(tx, client, logger)
 			if err != nil {
 				return err
 			}
@@ -202,7 +202,7 @@ func doJobs(playbook *def.Playbook, args *def.DeployArgs, client *def.Client, lo
 			if err != nil {
 				return err
 			}
-			job.Result, err = RegisterNameJob(job.RegisterName, args, playbook, txs, client, logger)
+			job.Result, err = RegisterNameJob(txs, client, logger)
 			if err != nil {
 				return err
 			}
@@ -212,7 +212,7 @@ func doJobs(playbook *def.Playbook, args *def.DeployArgs, client *def.Client, lo
 			if err != nil {
 				return err
 			}
-			job.Result, err = PermissionJob(job.Permission, playbook.Account, tx, client, logger)
+			job.Result, err = PermissionJob(tx, client, logger)
 			if err != nil {
 				return err
 			}
@@ -222,7 +222,7 @@ func doJobs(playbook *def.Playbook, args *def.DeployArgs, client *def.Client, lo
 			if err != nil {
 				return err
 			}
-			job.Result, err = IdentifyJob(job.Identify, tx, playbook.Account, client, logger)
+			job.Result, err = IdentifyJob(tx, client, logger)
 			if err != nil {
 				return err
 			}
@@ -234,15 +234,15 @@ func doJobs(playbook *def.Playbook, args *def.DeployArgs, client *def.Client, lo
 			if ferr != nil {
 				return ferr
 			}
-			job.Result, err = DeployJob(job.Deploy, args, playbook, client, txs, contracts, logger)
+			job.Result, err = DeployJob(job.Deploy, playbook, client, txs, contracts, logger)
 
 		case *def.Call:
 			announce(job.Name, "Call", logger)
 			CallTx, ferr := FormulateCallJob(job.Call, args, playbook, client, logger)
 			if ferr != nil {
 				return ferr
 			}
-			job.Result, job.Variables, err = CallJob(job.Call, CallTx, args, playbook, client, logger)
+			job.Result, job.Variables, err = CallJob(job.Call, CallTx, playbook, client, logger)
 		case *def.Build:
 			announce(job.Name, "Build", logger)
 			var resp *compilers.Response

deploy/jobs/jobs_contracts.go

@@ -19,7 +19,7 @@ import (
 	hex "github.com/tmthrgd/go-hex"
 )
 
-var errCodeMissing = fmt.Errorf("error: no binary code found in contract. Contract may be abstract due to missing function body or inherited function signatures not matching.")
+var errCodeMissing = fmt.Errorf("error: no binary code found in contract. Contract may be abstract due to missing function body or inherited function signatures not matching")
 
 func BuildJob(build *def.Build, deployScript *def.Playbook, resp *compilers.Response, logger *logging.Logger) (result string, err error) {
 	// assemble contract
@@ -46,10 +46,8 @@ func BuildJob(build *def.Build, deployScript *def.Playbook, resp *compilers.Resp
 	if binP == "" {
 		binP = deployScript.BinPath
 	} else {
-		if _, err := os.Stat(binP); os.IsNotExist(err) {
-			if err := os.Mkdir(binP, 0775); err != nil {
-				return "", err
-			}
+		if err := os.MkdirAll(binP, 0775); err != nil {
+			return "", err
 		}
 	}
 
@@ -79,10 +77,8 @@ func BuildJob(build *def.Build, deployScript *def.Playbook, resp *compilers.Resp
 			dir := filepath.Dir(build.Store)
 			file := filepath.Base(build.Store)
 
-			if _, err := os.Stat(dir); os.IsNotExist(err) {
-				if err := os.Mkdir(dir, 0775); err != nil {
-					return "", err
-				}
+			if err := os.MkdirAll(dir, 0775); err != nil {
+				return "", err
 			}
 
 			err = res.Contract.Save(dir, file)
@@ -275,17 +271,17 @@ func FormulateDeployJob(deploy *def.Deploy, do *def.DeployArgs, deployScript *de
 	return
 }
 
-func DeployJob(deploy *def.Deploy, do *def.DeployArgs, script *def.Playbook, client *def.Client, txs []*payload.CallTx, contracts []*compilers.ResponseItem, logger *logging.Logger) (result string, err error) {
+func DeployJob(deploy *def.Deploy, script *def.Playbook, client *def.Client, txs []*payload.CallTx, contracts []*compilers.ResponseItem, logger *logging.Logger) (result string, err error) {
 	// saving contract
 	// additional data may be sent along with the contract
 	// these are naively added to the end of the contract code using standard
 	// mint packing
 
 	for i, tx := range txs {
 		// Sign, broadcast, display
-		contractAddress, err := deployFinalize(do, client, tx, logger)
+		contractAddress, err := deployFinalize(client, tx, logger)
 		if err != nil {
-			return "", fmt.Errorf("Error finalizing contract deploy %s: %v", deploy.Contract, err)
+			return "", fmt.Errorf("error finalizing contract deploy %s: %w", deploy.Contract, err)
 		}
 
 		// saving contract/library abi at abi/address
@@ -467,8 +463,8 @@ func FormulateCallJob(call *def.Call, do *def.DeployArgs, deployScript *def.Play
 			packedBytes, funcSpec, err = abi.EncodeFunctionCallFromFile(call.Destination, deployScript.BinPath, call.Function, logger, callDataArray...)
 		}
 		if err != nil {
-			err = util.ABIErrorHandler(err, call, nil, logger)
-			return
+			return nil, fmt.Errorf("error in FormulateCallJob with %v: %w",
+				call, err)
 		}
 	}
 
@@ -494,14 +490,12 @@ func FormulateCallJob(call *def.Call, do *def.DeployArgs, deployScript *def.Play
 	}, logger)
 }
 
-func CallJob(call *def.Call, tx *payload.CallTx, do *def.DeployArgs, playbook *def.Playbook, client *def.Client, logger *logging.Logger) (string, []*abi.Variable, error) {
-	var err error
+func CallJob(call *def.Call, tx *payload.CallTx, playbook *def.Playbook, client *def.Client, logger *logging.Logger) (string, []*abi.Variable, error) {
 
 	// Sign, broadcast, display
 	txe, err := client.SignAndBroadcast(tx, logger)
 	if err != nil {
-		var err = util.ChainErrorHandler(payload.InputsString(tx.GetInputs()), err, logger)
-		return "", nil, err
+		return "", nil, fmt.Errorf("error in CallJob with %v: %w", call, err)
 	}
 
 	if txe.Exception != nil {
@@ -561,16 +555,14 @@ func CallJob(call *def.Call, tx *payload.CallTx, do *def.DeployArgs, playbook *d
 	return result, call.Variables, nil
 }
 
-func deployFinalize(do *def.DeployArgs, client *def.Client, tx payload.Payload, logger *logging.Logger) (*crypto.Address, error) {
+func deployFinalize(client *def.Client, tx payload.Payload, logger *logging.Logger) (*crypto.Address, error) {
 	txe, err := client.SignAndBroadcast(tx, logger)
 	if err != nil {
-		return nil, util.ChainErrorHandler(payload.InputsString(tx.GetInputs()), err, logger)
-	}
-
-	if err := util.ReadTxSignAndBroadcast(txe, err, logger); err != nil {
 		return nil, err
 	}
 
+	LogTxExecution(txe, logger)
+
 	// The contructor can generate events
 	logEvents(txe, client, logger)
 

deploy/jobs/jobs_governance.go

@@ -40,6 +40,11 @@ func FormulateUpdateAccountJob(gov *def.UpdateAccount, account string, client *d
 		arg.Address = gov.Target
 	} else {
 		arg.PublicKey = gov.Target
+		pk, err := def.PublicKeyFromString(gov.Target)
+		if err != nil {
+			panic(err)
+		}
+		arg.Address = pk.GetAddress().String()
 	}
 
 	tx, err := client.UpdateAccount(arg, logger)
@@ -50,13 +55,13 @@ func FormulateUpdateAccountJob(gov *def.UpdateAccount, account string, client *d
 	return tx, util.Variables(arg), nil
 }
 
-func UpdateAccountJob(gov *def.UpdateAccount, account string, tx *payload.GovTx, client *def.Client, logger *logging.Logger) error {
+func UpdateAccountJob(tx *payload.GovTx, client *def.Client, logger *logging.Logger) error {
 	txe, err := client.SignAndBroadcast(tx, logger)
 	if err != nil {
-		return util.ChainErrorHandler(account, err, logger)
+		return fmt.Errorf("error in UpdateAccountJob with payload %v: %w", tx, err)
 	}
 
-	util.ReadTxSignAndBroadcast(txe, err, logger)
+	LogTxExecution(txe, logger)
 	if err != nil {
 		return err
 	}
@@ -82,14 +87,14 @@ func FormulateBondJob(bond *def.Bond, account string, client *def.Client, logger
 	return client.Bond(arg, logger)
 }
 
-func BondJob(bond *def.Bond, tx *payload.BondTx, account string, client *def.Client, logger *logging.Logger) (string, error) {
+func BondJob(tx *payload.BondTx, client *def.Client, logger *logging.Logger) (string, error) {
 	// Sign, broadcast, display
 	txe, err := client.SignAndBroadcast(tx, logger)
 	if err != nil {
-		return "", util.ChainErrorHandler(account, err, logger)
+		return "", fmt.Errorf("error in BondJob with payload %v: %w", tx, err)
 	}
 
-	util.ReadTxSignAndBroadcast(txe, err, logger)
+	LogTxExecution(txe, logger)
 	if err != nil {
 		return "", err
 	}
@@ -114,14 +119,14 @@ func FormulateUnbondJob(unbond *def.Unbond, account string, client *def.Client,
 	return client.Unbond(arg, logger)
 }
 
-func UnbondJob(bond *def.Unbond, tx *payload.UnbondTx, account string, client *def.Client, logger *logging.Logger) (string, error) {
+func UnbondJob(tx *payload.UnbondTx, client *def.Client, logger *logging.Logger) (string, error) {
 	// Sign, broadcast, display
 	txe, err := client.SignAndBroadcast(tx, logger)
 	if err != nil {
-		return "", util.ChainErrorHandler(account, err, logger)
+		return "", fmt.Errorf("error in UnbondJob with payload %v: %w", tx, err)
 	}
 
-	util.ReadTxSignAndBroadcast(txe, err, logger)
+	LogTxExecution(txe, logger)
 	if err != nil {
 		return "", err
 	}

deploy/jobs/jobs_proposal.go

@@ -242,8 +242,7 @@ func ProposalJob(prop *def.Proposal, do *def.DeployArgs, parentScript *def.Playb
 
 	txe, err := client.SignAndBroadcast(proposalTx, logger)
 	if err != nil {
-		var err = util.ChainErrorHandler(proposalTx.Input.Address.String(), err, logger)
-		return "", err
+		return "", fmt.Errorf("error in ProposalJob with proposal %v: %w", prop, err)
 	}
 
 	result := fmt.Sprintf("%X", txe.Receipt.TxHash)

deploy/jobs/jobs_test_jobs.go

@@ -34,8 +34,8 @@ func QueryContractJob(query *def.QueryContract, do *def.DeployArgs, script *def.
 		data = hex.EncodeToString(packedBytes)
 	}
 	if err != nil {
-		var err = util.ABIErrorHandler(err, nil, query, logger)
-		return "", nil, err
+		return "", nil, fmt.Errorf("error in QueryContractJob with %v: %w",
+			query, err)
 	}
 
 	logger.InfoMsg("Query contract",