Parallel processing with ordered output in Go. This module processes work concurrently / in parallel and returns output in a channel in the order of input.
go get github.com/tejzpr/ordered-concurrently
import concurrently "github.com/tejzpr/ordered-concurrently"
// The work that needs to be performed
func workFn(val interface{}) interface{} {
time.Sleep(time.Millisecond * time.Duration(rand.Intn(100)))
return val.(int) * 2
}
func main() {
max := 100
// Can be a non blocking channel as well
inputChan := make(chan *concurrently.OrderedInput)
wg := &sync.WaitGroup{}
outChan := concurrently.Process(inputChan, workFn, &concurrently.Options{PoolSize: 10})
go func() {
for out := range outChan {
log.Println(out.Value)
wg.Done()
}
}()
// Create work and sent to input channel
// Output will be in the order of input
for work := 0; work < max; work++ {
wg.Add(1)
input := &concurrently.OrderedInput{work}
inputChan <- input
}
close(inputChan)
wg.Wait()
}
Thanks to u/justinisrael for inputs on improving resource usage.