Change from bash script to flag on multi.

cmd/multi.go

@@ -56,6 +56,7 @@ var (
 	partialDirMerge     string
 	partialDirClean     bool
 	createPartial       bool
+	finishPartial       bool
 	multiInodes         int
 	multiStatJobs       int
 	multiCh             string
@@ -161,6 +162,7 @@ func init() {
 		"from specified directory after merging")
 	multiCmd.Flags().BoolVarP(&createPartial, "create_partial_dir", "p", false, "perform the walk, "+
 		"stat, and combine steps only")
+	multiCmd.Flags().BoolVarP(&finishPartial, "partial_dir_finish", "c", false, "perform the basedir and tidy step on a partial run")
 	multiCmd.Flags().IntVarP(&multiInodes, "inodes_per_stat", "n",
 		defaultInodesPerJob, "number of inodes per parallel stat job")
 	multiCmd.Flags().IntVarP(&multiStatJobs, "num_stat_jobs", "j",
@@ -215,17 +217,19 @@ func doMultiScheduling(args []string, sudo bool) error {
 		return err
 	}
 
-	scheduleWalkJobs(outputRoot, args, unique, multiStatJobs, multiInodes, multiCh, forcedQueue, s)
+	if !finishPartial {
+		scheduleWalkJobs(outputRoot, args, unique, multiStatJobs, multiInodes, multiCh, forcedQueue, s)
 
-	if partialDirMerge != "" {
-		unique = scheduleStaticCopy(outputRoot, unique, partialDirMerge, partialDirClean, s)
-	}
+		if partialDirMerge != "" {
+			unique = scheduleStaticCopy(outputRoot, unique, partialDirMerge, partialDirClean, s)
+		}
 
-	if createPartial {
-		s.DisableSudo()
-		schedulePartialSentinel(outputRoot, unique, s)
+		if createPartial {
+			s.DisableSudo()
+			schedulePartialSentinel(outputRoot, unique, s)
 
-		return nil
+			return nil
+		}
 	}
 
 	scheduleBasedirsJob(outputRoot, unique, s)
@@ -349,7 +353,8 @@ func copyReqs() *jqs.Requirements {
 }
 
 func scheduleStaticCopy(outputRoot, unique, partialDirMerge string, partialDirClean bool,
-	s *scheduler.Scheduler) string {
+	s *scheduler.Scheduler,
+) string {
 	var remove string
 
 	if partialDirClean {

main_test.go

@@ -192,11 +192,11 @@ func multiTests(t *testing.T, subcommand ...string) {
 	date := time.Now().Format("20060102")
 
 	Convey("A partial 'wrstat multi' command produces the correct jobs to run, with sudo enabled", func() {
-		testPartial(t, true, subcommand, date, walkReqs, touchReqs, combineReqs)
+		testPartial(t, true, subcommand, date, walkReqs, touchReqs, combineReqs, baseDirsReqs, tidyReqs)
 	})
 
 	Convey("A partial 'wrstat multi' command produces the correct jobs to run, with sudo not enabled", func() {
-		testPartial(t, false, subcommand, date, walkReqs, touchReqs, combineReqs)
+		testPartial(t, false, subcommand, date, walkReqs, touchReqs, combineReqs, baseDirsReqs, tidyReqs)
 	})
 
 	Convey("A full 'wrstat multi' command produces the correct jobs to run", func() {
@@ -430,7 +430,8 @@ func multiTests(t *testing.T, subcommand ...string) {
 }
 
 func testPartial(t *testing.T, sudo bool, subcommand []string, date string, walkReqs,
-	touchReqs, combineReqs *scheduler.Requirements) {
+	touchReqs, combineReqs, baseDirsReqs, tidyReqs *scheduler.Requirements,
+) {
 	t.Helper()
 
 	workingDir := t.TempDir()
@@ -533,6 +534,51 @@ func testPartial(t *testing.T, sudo bool, subcommand []string, date string, walk
 	}
 
 	So(jobs, ShouldResemble, expectation)
+
+	Convey("…finishing the partial run runs the correct jobs", func() {
+		workingDir := t.TempDir()
+		_, _, jobs, err := runWRStat(append(subcommand, "-w", workingDir, "-f", "final_output", "-q", "quota_file",
+			"-o", "owners_file", "-c", "/some/path", "/some-other/path")...)
+		So(err, ShouldBeNil)
+
+		So(len(jobs), ShouldEqual, 2)
+
+		repGroup := jobs[0].RepGroup[len(jobs[0].RepGroup)-20:]
+		expectation := []*jobqueue.Job{
+			{
+				Cmd: fmt.Sprintf("%s basedir -q \"quota_file\" -o \"owners_file\"  \"%s/%s\" \"final_output\"",
+					exe, workingDir, repGroup),
+				CwdMatters:   true,
+				RepGroup:     fmt.Sprintf("wrstat-basedir-%s-%s", date, repGroup),
+				ReqGroup:     "wrstat-basedir",
+				Requirements: baseDirsReqs,
+				Override:     1,
+				Retries:      30,
+				DepGroups:    []string{repGroup + ".basedir"},
+				Dependencies: jobqueue.Dependencies{
+					{
+						DepGroup: repGroup,
+					},
+				},
+			},
+			{
+				Cmd:          fmt.Sprintf("%s tidy -f final_output -d %s %s/%s", exe, date, workingDir, repGroup),
+				CwdMatters:   true,
+				RepGroup:     fmt.Sprintf("wrstat-tidy-final_output-%s-%s", date, repGroup),
+				ReqGroup:     "wrstat-tidy",
+				Requirements: tidyReqs,
+				Override:     1,
+				Retries:      30,
+				Dependencies: jobqueue.Dependencies{
+					{
+						DepGroup: repGroup + ".basedir",
+					},
+				},
+			},
+		}
+
+		So(jobs, ShouldResemble, expectation)
+	})
 }
 
 func TestMulti(t *testing.T) {
@@ -588,8 +634,10 @@ func TestWalk(t *testing.T) {
 		So(jobs, ShouldResemble, jobsExpectation)
 
 		expected := ""
-		for _, subPath := range []string{"", "/a", "/a/b", "/a/b/c", "/a/b/c/d", "/a/b/c/d/e",
-			"/a/b/c/test.txt", "/a/b/f", "/a/b/f/tes\nt2.csv", "/a/g", "/a/g/h", "/a/test3"} {
+		for _, subPath := range []string{
+			"", "/a", "/a/b", "/a/b/c", "/a/b/c/d", "/a/b/c/d/e",
+			"/a/b/c/test.txt", "/a/b/f", "/a/b/f/tes\nt2.csv", "/a/g", "/a/g/h", "/a/test3",
+		} {
 			expected += encode.Base64Encode(tmp+subPath) + "\n"
 		}