Added parallel computing

master
Paulo Simão 2021-10-24 21:42:20 -03:00
parent 8ca9e4b50c
commit da1957a8c6
3 changed files with 114 additions and 36 deletions

View File

@ -4,9 +4,9 @@ An abstraction for shell directly in GO
## Goal
Simplify process spawning and execution on top of exec.Command function.
Simplify process spawning and execution on top of exec.Command function.
The proposal is: use a string based approach for launching other process - as if you were calling a shell, but instead,
The proposal is: use a string based approach for launching other process - as if you were calling a shell, but instead,
its go directly executing your commands.
Example:
@ -19,12 +19,13 @@ import "go.digitalcircle.com.br/open/shelly"
import "os"
import "os/exec"
func main(){
func main() {
err := shelly.Exec(`
ls -larth
pwd
ls -larth &
pwd &
whoami
date`, &shelly.Opts{
Await: false,
SetupProc: func(cmd *exec.Cmd) {
cmd.Stdout = os.Stdout
cmd.Stderr = os.Stderr
@ -36,5 +37,8 @@ func main(){
}
```
Please note: Commands may be connected by `;` and `\n`. In any case they will be executed sequentially.
`&` will come later (pending implementation), and will allow processes to run in parallel.
Please note: Commands may be connected by `;` and `\n`. In any case they will be executed sequentially. Lines terminated
by `&` will be executed in parallel.
In case the Await property is true, shelly will wait for all lines to be executed (including parallel ones), otherwise
it will return as soon as serial tasks are finished (if any)

123
lib.go
View File

@ -6,14 +6,11 @@ import (
"log"
"os/exec"
"strings"
"sync"
)
type Mode int
type Opts struct {
SetupProc func(cmd *exec.Cmd)
}
func (m Mode) String() string {
switch m {
case MODE_NORMAL:
@ -36,6 +33,29 @@ const (
MODE_ESCAPE_STR
)
type LineType int
const (
LINETYPE_SERIAL LineType = iota
LINETYPE_PARALLEL
LINETYPE_COMMENT
)
// Opts tells Exec how to behave
// Opts.Await will inform whether its call should wait for all parallel started jobs to finish
// Opts.SetupProc will customize the processes before execution - Eg.: setting out and err
type Opts struct {
Await bool
SetupProc func(cmd *exec.Cmd)
}
// Line represents a line to be executed by the shell.
// Can be separated by \n our ; - in case of serial execution. & will be used to identify parallel execution
type Line struct {
Tokens []string
LType LineType
}
// Tokens will break the string into tokens - It can be later organized in command lines and alikes
func Tokens(str string) ([]string, error) {
params := make([]string, 0)
@ -127,25 +147,47 @@ func Tokens(str string) ([]string, error) {
}
// Lines will call Tokens first, and reorg them in executable lines
func Lines(str string) ([][]string, error) {
ret := make([][]string, 0)
func Lines(str string) ([]Line, error) {
ret := make([]Line, 0)
tokens, err := Tokens(str)
if err != nil {
return nil, err
}
line := make([]string, 0)
line := Line{
Tokens: make([]string, 0),
LType: LINETYPE_SERIAL,
}
for _, tk := range tokens {
switch tk {
case "\n", "&&", ";":
if len(line) > 0 {
case "\n", ";":
if len(line.Tokens) > 0 {
if strings.HasPrefix(line.Tokens[0], "#") {
line.LType = LINETYPE_COMMENT
} else {
line.LType = LINETYPE_SERIAL
}
ret = append(ret, line)
}
line = make([]string, 0)
line = Line{
Tokens: make([]string, 0),
}
case "&":
if len(line.Tokens) > 0 {
if strings.HasPrefix(line.Tokens[0], "#") {
line.LType = LINETYPE_COMMENT
} else {
line.LType = LINETYPE_PARALLEL
}
ret = append(ret, line)
}
line = Line{
Tokens: make([]string, 0),
}
default:
line = append(line, tk)
line.Tokens = append(line.Tokens, tk)
}
}
if len(line) > 0 {
if len(line.Tokens) > 0 {
ret = append(ret, line)
}
return ret, nil
@ -153,27 +195,56 @@ func Lines(str string) ([][]string, error) {
// Exec will call Lines and execute one by one
func Exec(str string, opts ...*Opts) error {
var opt *Opts
if opts != nil && len(opts) > 0 {
opt = opts[0]
}
if opt.SetupProc == nil {
opt.SetupProc = func(cmd *exec.Cmd) {
}
}
prepCmd := func(l Line) *exec.Cmd {
cmd := exec.Command(l.Tokens[0], l.Tokens[1:]...)
cmd.Stdout = log.Writer()
cmd.Stderr = log.Writer()
opt.SetupProc(cmd)
return cmd
}
wg := sync.WaitGroup{}
lines, err := Lines(str)
if err != nil {
return err
}
for _, l := range lines {
if len(l) < 1 {
switch l.LType {
case LINETYPE_COMMENT:
continue
case LINETYPE_SERIAL:
cmd := prepCmd(l)
err = cmd.Run()
if err != nil {
return err
}
case LINETYPE_PARALLEL:
go func(l Line) {
cmd := prepCmd(l)
wg.Add(1)
err = cmd.Run()
wg.Done()
}(l)
}
if strings.HasPrefix(l[0], "#") {
continue
}
cmd := exec.Command(l[0], l[1:]...)
if opts != nil && len(opts) > 0 {
opts[0].SetupProc(cmd)
}
cmd.Stdout = log.Writer()
cmd.Stderr = log.Writer()
err = cmd.Run()
if err != nil {
return err
}
}
if opt.Await {
wg.Wait()
}
return nil

View File

@ -28,10 +28,11 @@ func TestLines(t *testing.T) {
func TestExec(t *testing.T) {
err := Exec(`
ls -larth
pwd
whoami
ls -larth &
pwd &
whoami &
date`, &Opts{
Await: false,
SetupProc: func(cmd *exec.Cmd) {
cmd.Stdout = os.Stdout
cmd.Stderr = os.Stderr
@ -41,4 +42,6 @@ func TestExec(t *testing.T) {
t.Fatal(err)
}
log.Printf("DONE")
}