Added parallel computing

master
Paulo Simão 2021-10-24 21:42:20 -03:00
parent 8ca9e4b50c
commit da1957a8c6
3 changed files with 114 additions and 36 deletions

View File

@ -4,9 +4,9 @@ An abstraction for shell directly in GO
## Goal ## Goal
Simplify process spawning and execution on top of exec.Command function. Simplify process spawning and execution on top of exec.Command function.
The proposal is: use a string based approach for launching other process - as if you were calling a shell, but instead, The proposal is: use a string based approach for launching other process - as if you were calling a shell, but instead,
its go directly executing your commands. its go directly executing your commands.
Example: Example:
@ -19,12 +19,13 @@ import "go.digitalcircle.com.br/open/shelly"
import "os" import "os"
import "os/exec" import "os/exec"
func main(){ func main() {
err := shelly.Exec(` err := shelly.Exec(`
ls -larth ls -larth &
pwd pwd &
whoami whoami
date`, &shelly.Opts{ date`, &shelly.Opts{
Await: false,
SetupProc: func(cmd *exec.Cmd) { SetupProc: func(cmd *exec.Cmd) {
cmd.Stdout = os.Stdout cmd.Stdout = os.Stdout
cmd.Stderr = os.Stderr cmd.Stderr = os.Stderr
@ -36,5 +37,8 @@ func main(){
} }
``` ```
Please note: Commands may be connected by `;` and `\n`. In any case they will be executed sequentially. Please note: Commands may be connected by `;` and `\n`. In any case they will be executed sequentially. Lines terminated
`&` will come later (pending implementation), and will allow processes to run in parallel. by `&` will be executed in parallel.
In case the Await property is true, shelly will wait for all lines to be executed (including parallel ones), otherwise
it will return as soon as serial tasks are finished (if any)

123
lib.go
View File

@ -6,14 +6,11 @@ import (
"log" "log"
"os/exec" "os/exec"
"strings" "strings"
"sync"
) )
type Mode int type Mode int
type Opts struct {
SetupProc func(cmd *exec.Cmd)
}
func (m Mode) String() string { func (m Mode) String() string {
switch m { switch m {
case MODE_NORMAL: case MODE_NORMAL:
@ -36,6 +33,29 @@ const (
MODE_ESCAPE_STR MODE_ESCAPE_STR
) )
type LineType int
const (
LINETYPE_SERIAL LineType = iota
LINETYPE_PARALLEL
LINETYPE_COMMENT
)
// Opts tells Exec how to behave
// Opts.Await will inform whether its call should wait for all parallel started jobs to finish
// Opts.SetupProc will customize the processes before execution - Eg.: setting out and err
type Opts struct {
Await bool
SetupProc func(cmd *exec.Cmd)
}
// Line represents a line to be executed by the shell.
// Can be separated by \n our ; - in case of serial execution. & will be used to identify parallel execution
type Line struct {
Tokens []string
LType LineType
}
// Tokens will break the string into tokens - It can be later organized in command lines and alikes // Tokens will break the string into tokens - It can be later organized in command lines and alikes
func Tokens(str string) ([]string, error) { func Tokens(str string) ([]string, error) {
params := make([]string, 0) params := make([]string, 0)
@ -127,25 +147,47 @@ func Tokens(str string) ([]string, error) {
} }
// Lines will call Tokens first, and reorg them in executable lines // Lines will call Tokens first, and reorg them in executable lines
func Lines(str string) ([][]string, error) { func Lines(str string) ([]Line, error) {
ret := make([][]string, 0) ret := make([]Line, 0)
tokens, err := Tokens(str) tokens, err := Tokens(str)
if err != nil { if err != nil {
return nil, err return nil, err
} }
line := make([]string, 0) line := Line{
Tokens: make([]string, 0),
LType: LINETYPE_SERIAL,
}
for _, tk := range tokens { for _, tk := range tokens {
switch tk { switch tk {
case "\n", "&&", ";": case "\n", ";":
if len(line) > 0 { if len(line.Tokens) > 0 {
if strings.HasPrefix(line.Tokens[0], "#") {
line.LType = LINETYPE_COMMENT
} else {
line.LType = LINETYPE_SERIAL
}
ret = append(ret, line) ret = append(ret, line)
} }
line = make([]string, 0) line = Line{
Tokens: make([]string, 0),
}
case "&":
if len(line.Tokens) > 0 {
if strings.HasPrefix(line.Tokens[0], "#") {
line.LType = LINETYPE_COMMENT
} else {
line.LType = LINETYPE_PARALLEL
}
ret = append(ret, line)
}
line = Line{
Tokens: make([]string, 0),
}
default: default:
line = append(line, tk) line.Tokens = append(line.Tokens, tk)
} }
} }
if len(line) > 0 { if len(line.Tokens) > 0 {
ret = append(ret, line) ret = append(ret, line)
} }
return ret, nil return ret, nil
@ -153,27 +195,56 @@ func Lines(str string) ([][]string, error) {
// Exec will call Lines and execute one by one // Exec will call Lines and execute one by one
func Exec(str string, opts ...*Opts) error { func Exec(str string, opts ...*Opts) error {
var opt *Opts
if opts != nil && len(opts) > 0 {
opt = opts[0]
}
if opt.SetupProc == nil {
opt.SetupProc = func(cmd *exec.Cmd) {
}
}
prepCmd := func(l Line) *exec.Cmd {
cmd := exec.Command(l.Tokens[0], l.Tokens[1:]...)
cmd.Stdout = log.Writer()
cmd.Stderr = log.Writer()
opt.SetupProc(cmd)
return cmd
}
wg := sync.WaitGroup{}
lines, err := Lines(str) lines, err := Lines(str)
if err != nil { if err != nil {
return err return err
} }
for _, l := range lines { for _, l := range lines {
if len(l) < 1 {
switch l.LType {
case LINETYPE_COMMENT:
continue continue
case LINETYPE_SERIAL:
cmd := prepCmd(l)
err = cmd.Run()
if err != nil {
return err
}
case LINETYPE_PARALLEL:
go func(l Line) {
cmd := prepCmd(l)
wg.Add(1)
err = cmd.Run()
wg.Done()
}(l)
} }
if strings.HasPrefix(l[0], "#") {
continue }
}
cmd := exec.Command(l[0], l[1:]...) if opt.Await {
if opts != nil && len(opts) > 0 { wg.Wait()
opts[0].SetupProc(cmd)
}
cmd.Stdout = log.Writer()
cmd.Stderr = log.Writer()
err = cmd.Run()
if err != nil {
return err
}
} }
return nil return nil

View File

@ -28,10 +28,11 @@ func TestLines(t *testing.T) {
func TestExec(t *testing.T) { func TestExec(t *testing.T) {
err := Exec(` err := Exec(`
ls -larth ls -larth &
pwd pwd &
whoami whoami &
date`, &Opts{ date`, &Opts{
Await: false,
SetupProc: func(cmd *exec.Cmd) { SetupProc: func(cmd *exec.Cmd) {
cmd.Stdout = os.Stdout cmd.Stdout = os.Stdout
cmd.Stderr = os.Stderr cmd.Stderr = os.Stderr
@ -41,4 +42,6 @@ func TestExec(t *testing.T) {
t.Fatal(err) t.Fatal(err)
} }
log.Printf("DONE")
} }